feat: add Watchdog recovery and drift rebuild workflows
This commit is contained in:
@@ -54,6 +54,19 @@ async function tick(seconds = 15, players = 4, healthy = true) {
|
||||
const scriptPath = (name: string) => join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/test.gsc');
|
||||
|
||||
describe('durable beta test plans', () => {
|
||||
it('waits for occupied telemetry to clear before restarting a beta despite an empty RCON response', async () => {
|
||||
const id = await openPlan(defaultPlan());
|
||||
const { listBetaServers } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await postTelemetry(gameEvent('heartbeat', 4, 2));
|
||||
await tick();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(listBetaServers(id)[0].stage).toBe('awaiting-empty');
|
||||
await postTelemetry(gameEvent('heartbeat', 4, 0));
|
||||
await tick();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('holds file deployment until each wave passes, and rollback leaves undeployed files alone', async () => {
|
||||
await writeFile(scriptPath(names[1]), 'original untracked file');
|
||||
const id = await openPlan({ ...defaultPlan(), waveSize: 1 }, names);
|
||||
@@ -81,7 +94,7 @@ describe('durable beta test plans', () => {
|
||||
it('uses script events instead of logs and keeps match progress across app restarts', async () => {
|
||||
await postTelemetry(gameEvent('game_end'));
|
||||
const id = await openPlan({ ...defaultPlan(), requirements: [requirement('matches', 'matches')] });
|
||||
await tick(); await tick();
|
||||
await tick(46); await tick();
|
||||
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
|
||||
await appendFile(join(fixture.t5, names[0], 'logs/games_sp.log'), 'InitGame:\n 0:00 J;new;1;Josh\nExitLevel: executed\n');
|
||||
await tick();
|
||||
@@ -98,7 +111,7 @@ describe('durable beta test plans', () => {
|
||||
it('automatically credits ordinary rounds only to active deployed beta servers', async () => {
|
||||
const id = await openPlan({ ...defaultPlan(), waveSize: 1, requirements: [requirement('round', 'round')] }, names);
|
||||
await postTelemetry(gameEvent('round_change'));
|
||||
await tick(); await tick();
|
||||
await tick(46); await tick();
|
||||
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
|
||||
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
|
||||
await postTelemetry(gameEvent('round_change', 99, 4, names[1]));
|
||||
@@ -117,7 +130,7 @@ describe('durable beta test plans', () => {
|
||||
const first = currentRun(id, names[0])!;
|
||||
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
|
||||
await postTelemetry(gameEvent('game_start'));
|
||||
await tick(); await tick();
|
||||
await tick(46); await tick();
|
||||
expect(currentRun(id, names[0])?.id).not.toBe(first.id);
|
||||
expect(currentRun(id, names[0])?.phase).toBe(1);
|
||||
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
|
||||
@@ -212,7 +225,7 @@ describe('durable beta test plans', () => {
|
||||
expect(retry.status).toBe(200);
|
||||
await postTelemetry(gameEvent('round_change'));
|
||||
expect(testView(id)?.servers).toHaveLength(0);
|
||||
await tick(); await tick();
|
||||
await tick(46); await tick();
|
||||
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
|
||||
await postTelemetry(gameEvent('round_change'));
|
||||
expect(testView(id)?.servers[0].progress[0].current).toBe(25);
|
||||
|
||||
@@ -435,7 +435,7 @@ describe('server creation map selection', () => {
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' });
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'not ready' });
|
||||
await processCreations();
|
||||
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', stage: 'Waiting for RCON health: timeout' });
|
||||
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', stage: 'Waiting for RCON or fresh telemetry health: timeout' });
|
||||
for (const map of [null, 'zombie_moon']) {
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' });
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'localhost', port: created.port }, rawReply: '', status: { players: 0, map, gametype: null, roster: [] } });
|
||||
|
||||
@@ -94,7 +94,7 @@ describe('shared creation queue', () => {
|
||||
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for Komodo health: starting' });
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, reason: 'timeout', endpoint: null, detail: 'booting' });
|
||||
await processCreations();
|
||||
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for RCON health: timeout' });
|
||||
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for RCON or fresh telemetry health: timeout' });
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce(status(jobs[0].name, 'zombie_moon'));
|
||||
await processCreations();
|
||||
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: expect.stringContaining('expected zombie_theater') });
|
||||
|
||||
@@ -73,6 +73,112 @@ describe('reviewed drift plans', () => {
|
||||
});
|
||||
|
||||
describe('drift scanning', () => {
|
||||
it.each([
|
||||
['script', 'moved'], ['script', 'unset'], ['plugin', 'moved'], ['plugin', 'unset']
|
||||
] as const)('detects formerly tracked %s files after their scope is %s', async (kind, change) => {
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { createScript, setScope } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
|
||||
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { destOfKind } = await import('../../src/lib/server/config.js');
|
||||
await seedServerRegistry();
|
||||
const server = 't5_zombie_theater_1';
|
||||
const relPath = kind === 'script' ? 'outdated.gsc' : 'outdated.dll';
|
||||
const { script, version } = createScript({ name: `outdated ${kind}`, kind, scopeType: 'game', scopeValue: 't5', targetRelPath: relPath, content: Buffer.from('previously promoted') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
|
||||
await runDriftDeployment({ servers: [server] });
|
||||
setScope({ scriptId: script.id, scopeType: change === 'moved' ? 'game' : null, scopeValue: change === 'moved' ? 't6' : null });
|
||||
expect(getServerScript(server, relPath)).toBeDefined();
|
||||
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ kind, relPath, driftKind: 'untracked' })]);
|
||||
await runDriftDeployment({ servers: [server] });
|
||||
expect(existsSync(join(destOfKind(server, kind), relPath))).toBe(false);
|
||||
expect(getServerScript(server, relPath)).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each(['script', 'plugin'] as const)('detects edited tracked %s bytes and restores their promoted version', async (kind) => {
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
|
||||
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { destOfKind } = await import('../../src/lib/server/config.js');
|
||||
await seedServerRegistry();
|
||||
const server = 't5_zombie_theater_1';
|
||||
const relPath = kind === 'script' ? 'managed.gsc' : 'managed.dll';
|
||||
const content = Buffer.from('promoted content');
|
||||
const { script, version } = createScript({ name: `managed ${kind}`, kind, scopeType: 'game', scopeValue: 't5', targetRelPath: relPath, content });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
|
||||
await runDriftDeployment({ servers: [server] });
|
||||
const inventory = getServerScript(server, relPath);
|
||||
const path = join(destOfKind(server, kind), relPath);
|
||||
await writeFile(path, 'changed outside Electric Cherry');
|
||||
expect(getServerScript(server, relPath)).toEqual(inventory);
|
||||
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath, kind, driftKind: 'replace' })]);
|
||||
await runDriftDeployment({ servers: [server] });
|
||||
expect(await readFile(path)).toEqual(content);
|
||||
expect(await scanServerDrift(server)).toEqual([]);
|
||||
});
|
||||
|
||||
it('compares composed config with disk even when inventory still has the promoted hash', async () => {
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js');
|
||||
await seedServerRegistry();
|
||||
const server = 't5_zombie_theater_1';
|
||||
const { script, version } = createScript({ name: 'baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
|
||||
await runDriftDeployment({ servers: [server] });
|
||||
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
|
||||
await writeFile(path, 'changed outside Electric Cherry');
|
||||
expect(await scanServerDrift(server)).toEqual([expect.objectContaining({ relPath: 'dedicated_sp.cfg', driftKind: 'replace' })]);
|
||||
await runDriftDeployment({ servers: [server] });
|
||||
expect(await readFile(path)).toEqual(getPromotedServerConfig('t5', 'zombie_theater'));
|
||||
});
|
||||
|
||||
it.each(['running', 'ready-promote'])('defers all drift for a server with a %s beta', async (state) => {
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await seedServerRegistry();
|
||||
const server = 't5_zombie_theater_1';
|
||||
const { script, version } = createScript({ name: 'baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set sv_voice "1"\n') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
|
||||
const experimental = uploadNewVersion(script.id, Buffer.from('set sv_voice "0"\n'));
|
||||
const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: experimental.id }], servers: [server], waitForHealth: false, waitForMatch: false });
|
||||
db().prepare('UPDATE betas SET state = ? WHERE id = ?').run(state, betaId);
|
||||
const path = join(fixture.t5, server, 'configs', 'dedicated_sp.cfg');
|
||||
const betaBytes = await readFile(path);
|
||||
const stray = join(fixture.t5, server, 'configs', 'stray.cfg');
|
||||
await writeFile(stray, 'also deferred');
|
||||
expect(await scanServerDrift(server)).toEqual([]);
|
||||
expect(await runDriftDeployment({ servers: [server] })).toMatchObject({ deploymentId: null, totalServers: 0 });
|
||||
expect(await readFile(path)).toEqual(betaBytes);
|
||||
expect(await readFile(stray, 'utf8')).toBe('also deferred');
|
||||
db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(betaId);
|
||||
expect((await scanServerDrift(server)).map(entry => entry.relPath)).toEqual(['dedicated_sp.cfg', 'stray.cfg']);
|
||||
});
|
||||
|
||||
it('reports an unavailable registered directory without hiding drift on another game root', async () => {
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const other = 't6_zombie_tranzit_1';
|
||||
await mkdir(join(fixture.t6, other, 'configs'), { recursive: true });
|
||||
await writeFile(join(fixture.t6, other, 'configs', 'stray.cfg'), 'untracked');
|
||||
await seedServerRegistry();
|
||||
await rm(fixture.t5, { recursive: true });
|
||||
await expect(scanServerDrift('t5_zombie_theater_1')).rejects.toThrow('Server files unavailable');
|
||||
await expect(runDriftDeployment({ servers: ['t5_zombie_theater_1'] })).rejects.toThrow('Server files unavailable');
|
||||
expect(existsSync(fixture.t5)).toBe(false);
|
||||
expect(db().prepare('SELECT * FROM deployments').all()).toEqual([]);
|
||||
expect(await scanServerDrift(other)).toEqual([expect.objectContaining({ server: other, relPath: 'stray.cfg', driftKind: 'untracked' })]);
|
||||
});
|
||||
|
||||
it('replaces a seeded config once instead of planning both removal and installation', async () => {
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
@@ -152,6 +258,47 @@ describe('drift scanning', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('drift rebuild campaigns', () => {
|
||||
it('captures intended changes without writing and follows the linked rebuild without another restart', async () => {
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { prepareDriftCampaign, createDriftRebuildCampaign } = await import('../../src/lib/server/fleet/drift.js');
|
||||
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { destOfKind } = await import('../../src/lib/server/config.js');
|
||||
await seedServerRegistry();
|
||||
const server = 't5_zombie_theater_1';
|
||||
const { script, version } = createScript({ name: 'managed', kind: 'script', scopeType: 'game', scopeValue: 't5', targetRelPath: 'managed.gsc', content: Buffer.from('promoted content') });
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
|
||||
const target = join(destOfKind(server, 'script'), 'managed.gsc');
|
||||
await writeFile(target, 'old content');
|
||||
const stray = join(destOfKind(server, 'config'), 'stray.cfg');
|
||||
await writeFile(stray, 'untracked');
|
||||
const prepared = await prepareDriftCampaign(server);
|
||||
const operationId = beginDeployment(server, 'rebuild', 'watchdog');
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(operationId);
|
||||
const campaignId = createDriftRebuildCampaign(server, operationId, 42, prepared);
|
||||
expect(campaignId).toBeTruthy();
|
||||
expect(await readFile(target, 'utf8')).toBe('old content');
|
||||
expect(await readFile(stray, 'utf8')).toBe('untracked');
|
||||
expect(db().prepare('SELECT rel_path, verb, content_before, content_after FROM deployment_files WHERE deployment_id = ? ORDER BY rel_path').all(campaignId))
|
||||
.toEqual([
|
||||
{ rel_path: 'managed.gsc', verb: 'updated', content_before: Buffer.from('old content'), content_after: version.content },
|
||||
{ rel_path: 'stray.cfg', verb: 'removed', content_before: Buffer.from('untracked'), content_after: null }
|
||||
]);
|
||||
await sweep();
|
||||
expect(db().prepare('SELECT action FROM server_deployments').all()).toEqual([{ action: 'rebuild' }]);
|
||||
expect(db().prepare('SELECT provisioning_id, restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaignId))
|
||||
.toEqual({ provisioning_id: operationId, restart_state: 'pending' });
|
||||
db().prepare("UPDATE server_deployments SET state = 'done', submitted_at = ?, finished_at = ? WHERE id = ?")
|
||||
.run(new Date().toISOString(), new Date().toISOString(), operationId);
|
||||
await sweep();
|
||||
expect(db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(campaignId)).toEqual({ restart_state: 'done' });
|
||||
expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get(campaignId)).toMatchObject({ finished_at: expect.any(String) });
|
||||
});
|
||||
});
|
||||
|
||||
// Deploys a promoted artifact to a scope-matching server, then deletes it
|
||||
// off disk so drift's "missing" pass has something to find.
|
||||
async function promoteAndOrphan(
|
||||
|
||||
@@ -29,10 +29,13 @@ beforeEach(async () => {
|
||||
// Exercise the upgrade path, with history and references present before migration.
|
||||
const old = new Database(fixture.dbPath);
|
||||
const schema = await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8');
|
||||
old.exec(schema.replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'"));
|
||||
old.exec(schema.replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'")
|
||||
.replace("'campaign','beta','maintenance','watchdog'", "'campaign','beta','maintenance'")
|
||||
.replace("server TEXT NOT NULL, -- Campaign observation survives a rebuild's registry gap.", 'server TEXT NOT NULL REFERENCES servers(name),'));
|
||||
old.prepare(`INSERT INTO server_deployments (rowid,id,server,action,state,stage,started_at,triggered_by) VALUES (42,'old','old-server','create','done','Complete','2026-09-01','test')`).run();
|
||||
old.prepare("INSERT INTO server_deployment_events (deployment_id,created_at,message) VALUES ('old','2026-09-01','Old activity')").run();
|
||||
old.prepare("INSERT INTO server_creation_jobs VALUES ('old','old-server',28961,'zombie','theater','zombie_theater')").run();
|
||||
old.prepare("INSERT INTO provisioning_sources VALUES ('old','campaign','legacy-campaign','old-server')").run();
|
||||
old.close();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`INSERT INTO servers (name,komodo_type,komodo_name,game,map,game_mode,map_slug,game_port,host_dir)
|
||||
@@ -73,6 +76,7 @@ describe('provisioning operations', () => {
|
||||
expect(db().prepare("SELECT rowid FROM server_deployments WHERE id = 'old'").get()).toEqual({ rowid: 42 });
|
||||
expect(db().prepare("SELECT message FROM server_deployment_events WHERE deployment_id = 'old'").get()).toEqual({ message: 'Old activity' });
|
||||
expect(db().prepare("SELECT server FROM server_creation_jobs WHERE deployment_id = 'old'").get()).toEqual({ server: 'old-server' });
|
||||
expect(db().prepare("SELECT kind, source_id FROM provisioning_sources WHERE operation_id = 'old'").get()).toEqual({ kind: 'campaign', source_id: 'legacy-campaign' });
|
||||
expect(db().pragma('foreign_key_check')).toEqual([]);
|
||||
expect(db().pragma('foreign_keys', { simple: true })).toBe(1);
|
||||
});
|
||||
@@ -244,3 +248,279 @@ describe('provisioning operations', () => {
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
async function incident(id = 'incident') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const now = new Date().toISOString();
|
||||
db().prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at)
|
||||
VALUES (?,?,'health','open','Server down',?,?)`).run(id, server, now, now);
|
||||
return id;
|
||||
}
|
||||
|
||||
async function report(players: number, timestamp = new Date().toISOString(), map = 'zombie_theater') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { gameEvent } = await import('../helpers/telemetry.js');
|
||||
db().prepare(`INSERT INTO server_telemetry (server,round,players,received_at,payload) VALUES (?,1,?,?,?)
|
||||
ON CONFLICT(server) DO UPDATE SET players=excluded.players, received_at=excluded.received_at, payload=excluded.payload`)
|
||||
.run(server, players, timestamp, JSON.stringify({ ...gameEvent('heartbeat', 1, players, server, map), receivedAt: timestamp }));
|
||||
}
|
||||
|
||||
describe('Watchdog provisioning and telemetry evidence', () => {
|
||||
it('keeps the original running intent when a queued removal is cancelled', async () => {
|
||||
const { destroyServer, cancelRemoval } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const removal = destroyServer(server);
|
||||
if (!removal.ok || !removal.deploymentId) throw new Error('Could not queue removal');
|
||||
expect(cancelRemoval(removal.deploymentId)).toBe(true);
|
||||
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'running' });
|
||||
});
|
||||
|
||||
it('preserves stop intent across process restarts, including an uncertain stop request', async () => {
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { finishDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const id = await queue('stop');
|
||||
finishDeployment(id, { ok: false, message: 'lost response' });
|
||||
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'stopped' });
|
||||
// Exercise a legacy upgrade where the stop exists but intent was not stored.
|
||||
db().prepare("UPDATE servers SET desired_state = 'running'").run();
|
||||
db().prepare("DELETE FROM settings WHERE key = 'watchdog_desired_state_v1'").run();
|
||||
closeDb();
|
||||
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'stopped' });
|
||||
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
await incident();
|
||||
expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toMatchObject({ ok: false, error: 'This server is intentionally stopped.' });
|
||||
await queue('start');
|
||||
expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'running' });
|
||||
});
|
||||
|
||||
it('deduplicates Watchdog requests, records attribution, and holds paused work', async () => {
|
||||
const { requestWatchdogLifecycle, getDeployment } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const result = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toEqual(result);
|
||||
expect(getDeployment(result.deploymentId)?.sources).toEqual([{ kind: 'watchdog', source_id: 'incident', server }]);
|
||||
expect(await operation(result.deploymentId)).toMatchObject({ triggered_by: 'watchdog:incident' });
|
||||
db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run();
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
db().prepare("UPDATE settings SET value = '1' WHERE key = 'watchdog_enabled'").run();
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('accepts fresh telemetry for occupancy and restart recovery without RCON, while rejecting old-process reports', async () => {
|
||||
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
vi.spyOn(config.rcon, 'password', 'get').mockReturnValue('');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
||||
vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'no-config', detail: 'No RCON', endpoint: null });
|
||||
await incident(); await report(0);
|
||||
const result = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await processProvisioning();
|
||||
expect(await operation(result.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
||||
await report(0);
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
await processProvisioning();
|
||||
expect(await operation(result.deploymentId)).toMatchObject({ state: 'done' });
|
||||
});
|
||||
|
||||
it('uses fresh occupied telemetry arriving during RCON to veto a zero-player restart', async () => {
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const id = await queue('restart');
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { await report(2); return 0; });
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(await operation(id)).toMatchObject({ state: 'waiting', players: 2 });
|
||||
await report(0);
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(3);
|
||||
await processProvisioning();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
expect(await operation(id)).toMatchObject({ state: 'waiting', players: 3 });
|
||||
});
|
||||
|
||||
it('rejects stale, future, pre-submission, and invalid telemetry evidence', async () => {
|
||||
const { telemetryEvidence, probePlayerCount } = await import('../../src/lib/server/fleet/status-evidence.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
||||
for (const offset of [-45_001, 1]) {
|
||||
await report(0, new Date(Date.now() + offset).toISOString());
|
||||
expect(telemetryEvidence(server)).toBeNull();
|
||||
expect(await probePlayerCount(server)).toBeNull();
|
||||
}
|
||||
await report(0);
|
||||
expect(telemetryEvidence(server, new Date().toISOString())).toBeNull();
|
||||
db().prepare("UPDATE server_telemetry SET payload = '{}' WHERE server = ?").run(server);
|
||||
expect(telemetryEvidence(server)).toBeNull();
|
||||
});
|
||||
|
||||
it('reserves a rebuild before transferring its exact restart slot and requires fresh emptiness', async () => {
|
||||
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoInspectHealthy, komodoDestroyDeployment, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await processProvisioning();
|
||||
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
|
||||
vi.setSystemTime(Date.now() + 300_000);
|
||||
const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toEqual(rebuild);
|
||||
expect(await operation(restart.deploymentId)).toMatchObject({ state: 'failed' });
|
||||
expect(db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ operation_id: rebuild.deploymentId, healthy_at: null });
|
||||
closeDb();
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
|
||||
vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'timeout', detail: 'No reply', endpoint: null });
|
||||
await processProvisioning();
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'waiting' });
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
|
||||
await processProvisioning();
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
|
||||
expect(db().prepare('SELECT 1 FROM restart_rollout WHERE server = ?').get(server)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('retains recovery capacity when a rebuild prerequisite fails or a different operation owns the slot', async () => {
|
||||
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { removeServerKey, saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
|
||||
removeServerKey(server);
|
||||
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
|
||||
expect(await operation(restart.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
||||
expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(server)).toEqual({ operation_id: restart.deploymentId });
|
||||
saveServerKey(server, 'saved-key');
|
||||
db().prepare("UPDATE restart_rollout SET operation_id = 'another-operation' WHERE server = ?").run(server);
|
||||
expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
|
||||
expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(server)).toEqual({ operation_id: 'another-operation' });
|
||||
expect(db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action = 'rebuild'").get()).toEqual({ n: 0 });
|
||||
});
|
||||
|
||||
it('does not let an in-flight old restart observation release a transferred rebuild slot', async () => {
|
||||
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { refreshRestartRollout } = await import('../../src/lib/server/provisioning/restarts.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
|
||||
let reportHealthy!: (value: { ok: true; state: string }) => void;
|
||||
vi.mocked(komodoInspectHealthy).mockImplementationOnce(() => new Promise(resolve => { reportHealthy = resolve; }));
|
||||
const observation = refreshRestartRollout();
|
||||
const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
reportHealthy({ ok: true, state: 'running' });
|
||||
await observation;
|
||||
expect(db().prepare('SELECT operation_id,healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ operation_id: rebuild.deploymentId, healthy_at: null });
|
||||
expect(await operation(restart.deploymentId)).toMatchObject({ state: 'failed' });
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'waiting' });
|
||||
});
|
||||
|
||||
it('resumes explicitly requested rebuild cleanup after the incident needs attention, even when Watchdog is paused', async () => {
|
||||
const { requestWatchdogLifecycle, resumeRebuild } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { deploymentStage, finishDeployment, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const rebuild = requestWatchdogLifecycle(server, 'rebuild', 'incident');
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
deploymentStage(rebuild.deploymentId, REMOVING_SERVER);
|
||||
finishDeployment(rebuild.deploymentId, { ok: false, message: 'Cleanup interrupted' });
|
||||
db().prepare("UPDATE watchdog_incidents SET state = 'attention' WHERE id = 'incident'").run();
|
||||
db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run();
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
||||
expect(resumeRebuild(rebuild.deploymentId)).toBe(true);
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['restart','rebuild'] as const)('hands retained failed-rebuild capacity to an explicit new-cycle %s attempt', async action => {
|
||||
const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { refreshRestartRollout } = await import('../../src/lib/server/provisioning/restarts.js');
|
||||
const { komodoDestroyDeployment, komodoInspectHealthy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
|
||||
if (!restart.ok) throw new Error(restart.error);
|
||||
await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
|
||||
const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
db().prepare("INSERT INTO watchdog_actions (id,incident_id,action,operation_id,created_at) VALUES ('failed-rebuild','incident','rebuild',?,?)")
|
||||
.run(rebuild.deploymentId, new Date().toISOString());
|
||||
vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Teardown failed' });
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'failed' });
|
||||
db().prepare("UPDATE watchdog_incidents SET cycle = 2, state = 'open' WHERE id = 'incident'").run();
|
||||
const retry = requestWatchdogLifecycle(server, action, 'incident');
|
||||
if (!retry.ok) throw new Error(retry.error);
|
||||
expect(db().prepare('SELECT operation_id,healthy_at FROM restart_rollout WHERE server = ?').get(server))
|
||||
.toEqual({ operation_id: retry.deploymentId, healthy_at: null });
|
||||
await refreshRestartRollout();
|
||||
// Old process health cannot release capacity before this retry submits.
|
||||
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
|
||||
if (action === 'rebuild') vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
|
||||
await processProvisioning();
|
||||
expect(await operation(retry.deploymentId)).toMatchObject({ state: action === 'restart' ? 'awaiting-health' : 'queued' });
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(action === 'restart' ? 2 : 1);
|
||||
});
|
||||
|
||||
it('keeps its linked drift campaign and file audit throughout rebuild, and verifies the replacement through telemetry', async () => {
|
||||
const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await incident();
|
||||
const rebuild = requestWatchdogLifecycle(server, 'rebuild', 'incident');
|
||||
if (!rebuild.ok) throw new Error(rebuild.error);
|
||||
const now = new Date().toISOString();
|
||||
db().prepare("INSERT INTO deployments (id,action,started_at) VALUES ('drift-audit','drift',?)").run(now);
|
||||
db().prepare("INSERT INTO deployment_servers (deployment_id,server,files_changed,restart_state,provisioning_id) VALUES ('drift-audit',?,1,'pending',?)").run(server, rebuild.deploymentId);
|
||||
db().prepare("INSERT INTO deployment_files (deployment_id,server,verb,rel_path) VALUES ('drift-audit',?,'updated','script.gsc')").run(server);
|
||||
db().prepare("INSERT INTO provisioning_sources VALUES (?,'campaign','drift-audit',?)").run(rebuild.deploymentId, server);
|
||||
db().prepare("INSERT INTO watchdog_actions (id,incident_id,action,operation_id,campaign_id,created_at) VALUES ('action','incident','rebuild',?,'drift-audit',?)").run(rebuild.deploymentId, now);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
|
||||
expect(db().prepare("SELECT provisioning_id FROM deployment_servers WHERE deployment_id = 'drift-audit'").get()).toEqual({ provisioning_id: rebuild.deploymentId });
|
||||
expect(db().prepare("SELECT COUNT(*) AS n FROM deployment_files WHERE deployment_id = 'drift-audit'").get()).toEqual({ n: 1 });
|
||||
expect(db().pragma('foreign_key_check')).toEqual([]);
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health' });
|
||||
vi.setSystemTime(Date.now() + 15_000); await report(0);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'timeout', detail: 'No reply', endpoint: null });
|
||||
await processProvisioning();
|
||||
expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'done' });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -183,9 +183,9 @@ it('still accepts and broadcasts telemetry if storing diagnostics fails', async
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
const { listTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
||||
expect(listTelemetry()).toEqual([{ ...payload, receivedAt: expect.any(String) }]);
|
||||
expect(listTelemetry()).toEqual([expect.objectContaining({ ...payload, receivedAt: expect.any(String) })]);
|
||||
expect(listener).toHaveBeenCalledWith({
|
||||
type: 'server-telemetry', entries: [{ ...payload, receivedAt: expect.any(String) }]
|
||||
type: 'server-telemetry', entries: [expect.objectContaining({ ...payload, receivedAt: expect.any(String) })]
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { access, mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { gameEvent } from '../helpers/telemetry.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(),
|
||||
komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
const server = 't5_zombie_theater_1';
|
||||
const healthyRcon = { ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } };
|
||||
const missingRcon = { ok: false as const, endpoint: { host: 'test', port: 28960 }, reason: 'timeout' as const, detail: 'No reply' };
|
||||
let fixture: Fixture;
|
||||
type Engine = typeof import('../../src/lib/server/watchdog/index.js');
|
||||
let watchdog: Engine;
|
||||
let database: typeof import('../../src/lib/server/db/index.js');
|
||||
let komodo: typeof import('../../src/lib/server/integrations/komodo.js');
|
||||
let rcon: typeof import('../../src/lib/server/integrations/rcon.js');
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.resetAllMocks();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-27T12:00:00Z'));
|
||||
fixture = await newFixture([server]);
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
|
||||
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
|
||||
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
|
||||
database = await import('../../src/lib/server/db/index.js');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
database.db().prepare('UPDATE servers SET game_port = 28960 WHERE name = ?').run(server);
|
||||
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
saveServerKey(server, 'saved-key');
|
||||
komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) {
|
||||
vi.mocked(komodo[name]).mockResolvedValue({ ok: true });
|
||||
}
|
||||
vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
|
||||
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon);
|
||||
watchdog = await import('../../src/lib/server/watchdog/index.js');
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); });
|
||||
|
||||
const incidents = () => watchdog.watchdogOverview().incidents;
|
||||
const onlyIncident = () => { expect(incidents()).toHaveLength(1); return incidents()[0]; };
|
||||
const operations = () => database.db().prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid').all() as { id: string; action: string; state: string }[];
|
||||
function advance(ms: number) { vi.setSystemTime(Date.now() + ms); }
|
||||
function down() { vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); }
|
||||
async function sustainedFailure() {
|
||||
down();
|
||||
await watchdog.watchdogTick(true);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]);
|
||||
return onlyIncident();
|
||||
}
|
||||
|
||||
describe('Watchdog detection and ownership', () => {
|
||||
it('defaults to automatic recovery and debounces manual scans without accelerating the failure window', async () => {
|
||||
expect(watchdog.watchdogOverview().enabled).toBe(true);
|
||||
down();
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(onlyIncident()).toMatchObject({ state: 'open', failures: 1 });
|
||||
for (let n = 0; n < 5; n++) await watchdog.watchdogTick(true);
|
||||
expect(onlyIncident().failures).toBe(1);
|
||||
expect(operations()).toEqual([]);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS);
|
||||
await watchdog.watchdogTick();
|
||||
expect(onlyIncident().failures).toBe(2);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS - 1);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([]);
|
||||
advance(1);
|
||||
await watchdog.watchdogTick(true);
|
||||
const row = onlyIncident();
|
||||
expect(row).toMatchObject({ state: 'recovering', failures: 3 });
|
||||
expect(watchdog.watchdogDetail(row.id)?.actions).toEqual([expect.objectContaining({ action: 'restart', operation_state: 'waiting' })]);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(1);
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps paused recovery observational and resumes one pending incident', async () => {
|
||||
watchdog.setWatchdogEnabled(false);
|
||||
down();
|
||||
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(onlyIncident()).toMatchObject({ state: 'open', failures: 4 });
|
||||
expect(operations()).toEqual([]);
|
||||
database.closeDb();
|
||||
expect(watchdog.watchdogOverview().enabled).toBe(false);
|
||||
watchdog.setWatchdogEnabled(true);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
watchdog.setWatchdogEnabled(false);
|
||||
await processProvisioning();
|
||||
expect(operations()[0].state).toBe('waiting');
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
watchdog.setWatchdogEnabled(true);
|
||||
await processProvisioning();
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not restart an unknown Komodo state even after repeated sustained observations', async () => {
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo request timed out' });
|
||||
for (let n = 0; n < 8; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open' });
|
||||
expect(operations()).toEqual([]);
|
||||
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
||||
down();
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(onlyIncident().failures).toBe(1);
|
||||
expect(operations()).toEqual([]);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([]);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
||||
});
|
||||
|
||||
it('allows a longer initial startup grace period before requesting recovery', async () => {
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
|
||||
for (let n = 0; n < 5; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(operations()).toEqual([]);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
|
||||
});
|
||||
|
||||
it('accepts fresh validated telemetry when RCON is unavailable and rechecks after it becomes stale', async () => {
|
||||
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
|
||||
saveTelemetry({ ...gameEvent('heartbeat', 17, 4, server), receivedAt: new Date().toISOString() });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
|
||||
advance(60_000);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 1 });
|
||||
saveTelemetry({ ...gameEvent('heartbeat', 18, 4, server), receivedAt: new Date().toISOString() });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(onlyIncident().state).toBe('resolved');
|
||||
expect(operations()).toEqual([]);
|
||||
});
|
||||
|
||||
it('preserves intentional stop intent after completion and database reopening', async () => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const stop = requestLifecycle(server, 'stop');
|
||||
expect(stop.ok).toBe(true);
|
||||
if (!stop.ok) throw new Error(stop.error);
|
||||
database.db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ? WHERE id = ?").run(new Date().toISOString(), stop.deploymentId);
|
||||
down();
|
||||
database.closeDb();
|
||||
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]);
|
||||
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['beta', 'campaign'] as const)('defers a server owned by %s work', async (owner) => {
|
||||
if (owner === 'beta') {
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { script, version } = createScript({ name: 'beta', content: Buffer.from('experimental'), targetRelPath: 'beta.gsc' });
|
||||
await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [server], waitForHealth: false, waitForMatch: false });
|
||||
} else {
|
||||
database.db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('campaign', 'promote', ?)").run(new Date().toISOString());
|
||||
database.db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('campaign', ?, 1, 'held')").run(server);
|
||||
}
|
||||
down();
|
||||
for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(operations()).toEqual([]);
|
||||
expect(komodo.komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rechecks lifecycle ownership after a health probe yields to an operator stop', async () => {
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockImplementationOnce(async () => {
|
||||
expect(requestLifecycle(server, 'stop').ok).toBe(true);
|
||||
return { ok: false, state: 'exited', reason: 'Container exited' };
|
||||
});
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(incidents()).toEqual([]);
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Watchdog recovery persistence', () => {
|
||||
it('resolves an attention incident after an intentional successful destroy', async () => {
|
||||
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
removeServerKey(server);
|
||||
await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
|
||||
await watchdog.watchdogTick(true);
|
||||
const id = onlyIncident().id;
|
||||
expect(onlyIncident().state).toBe('attention');
|
||||
expect(destroyServer(server)).toMatchObject({ ok: true });
|
||||
down();
|
||||
await processDestructions();
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'destroy', state: 'done' })]);
|
||||
expect(database.db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(id)?.incident).toMatchObject({ state: 'resolved', finished_at: expect.any(String) });
|
||||
expect(watchdog.watchdogDetail(id)?.events.at(-1)?.message).toMatch(/destroyed|registered|retired/i);
|
||||
});
|
||||
|
||||
it('requests one restart, escalates once to rebuild, and retains exhausted attention across reopening', async () => {
|
||||
const row = await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(operations()[0].state).toBe('awaiting-health');
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS - 1);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(1);
|
||||
advance(1);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
||||
expect(watchdog.watchdogDetail(row.id)?.actions).toHaveLength(2);
|
||||
vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: false, kind: 'unreachable', message: 'Host unavailable' });
|
||||
await processProvisioning();
|
||||
expect(operations()[1].state).toBe('failed');
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('attention');
|
||||
database.closeDb();
|
||||
for (let n = 0; n < 4; n++) { advance(watchdog.WATCHDOG_RECOVERY_MS); await watchdog.watchdogTick(true); }
|
||||
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('attention');
|
||||
expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not escalate a timed out restart while automatic recovery is paused', async () => {
|
||||
await sustainedFailure();
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
await processProvisioning();
|
||||
watchdog.setWatchdogEnabled(false);
|
||||
advance(watchdog.WATCHDOG_RECOVERY_MS);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toHaveLength(1);
|
||||
watchdog.setWatchdogEnabled(true);
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
|
||||
});
|
||||
|
||||
it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => {
|
||||
down();
|
||||
await watchdog.watchdogTick(true);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
const id = onlyIncident().id;
|
||||
database.closeDb();
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
expect(onlyIncident().id).toBe(id);
|
||||
const queued = operations();
|
||||
expect(queued).toHaveLength(1);
|
||||
database.closeDb();
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual(queued);
|
||||
expect(watchdog.watchdogDetail(id)?.actions).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('permits an explicit new attempt after dismissal without silently rearming a sustained condition', async () => {
|
||||
down();
|
||||
await watchdog.watchdogTick(true);
|
||||
const id = onlyIncident().id;
|
||||
expect(watchdog.updateWatchdogIncident(id, 'dismiss')).toEqual({ ok: true });
|
||||
for (let n = 0; n < 4; n++) { advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); }
|
||||
expect(operations()).toEqual([]);
|
||||
expect(onlyIncident().state).toBe('dismissed');
|
||||
expect(watchdog.updateWatchdogIncident(id, 'retry')).toEqual({ ok: true });
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([]);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
expect(onlyIncident()).toMatchObject({ id, state: 'recovering', cycle: 2 });
|
||||
expect(watchdog.watchdogDetail(id)?.actions).toEqual([expect.objectContaining({ action: 'restart', cycle: 2 })]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Watchdog APIs', () => {
|
||||
const request = (body: unknown, raw = false) => new Request('http://localhost/api/watchdog', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: raw ? String(body) : JSON.stringify(body)
|
||||
});
|
||||
|
||||
it('returns overview state and validates scan actions and pause booleans', async () => {
|
||||
const { GET, POST } = await import('../../src/routes/api/watchdog/+server.js');
|
||||
expect(await GET().json()).toEqual({ enabled: true, lastScanAt: null, scanError: null, incidents: [] });
|
||||
expect((await POST({ request: request('{', true) })).status).toBe(400);
|
||||
for (const body of [null, [], {}, { action: 'restart' }, { enabled: 'false' }, { enabled: 0 }, { action: 'scan', enabled: 'false' }]) {
|
||||
expect((await POST({ request: request(body) })).status).toBe(400);
|
||||
}
|
||||
expect(await (await POST({ request: request({ enabled: false }) })).json()).toMatchObject({ enabled: false });
|
||||
database.closeDb();
|
||||
expect(await GET().json()).toMatchObject({ enabled: false });
|
||||
down();
|
||||
const scanned = await POST({ request: request({ action: 'scan' }) });
|
||||
expect(scanned.status).toBe(200);
|
||||
expect(await scanned.json()).toMatchObject({ enabled: false, lastScanAt: expect.any(String), incidents: [expect.objectContaining({ kind: 'health', state: 'open' })] });
|
||||
expect(operations()).toEqual([]);
|
||||
});
|
||||
|
||||
it('validates incident actions and refuses retries while provisioning is pending', async () => {
|
||||
const { GET, POST } = await import('../../src/routes/api/watchdog/[id]/+server.js');
|
||||
expect(GET({ params: { id: 'missing' } }).status).toBe(404);
|
||||
expect((await POST({ params: { id: 'missing' }, request: request({ action: 'retry' }) })).status).toBe(404);
|
||||
for (const body of [null, {}, [], { action: 'restart' }]) {
|
||||
expect((await POST({ params: { id: 'missing' }, request: request(body) })).status).toBe(400);
|
||||
}
|
||||
expect((await POST({ params: { id: 'missing' }, request: request('{', true) })).status).toBe(400);
|
||||
const row = await sustainedFailure();
|
||||
expect(await GET({ params: { id: row.id } }).json()).toMatchObject({
|
||||
incident: { id: row.id, state: 'recovering' }, actions: [expect.objectContaining({ action: 'restart' })], events: expect.any(Array)
|
||||
});
|
||||
for (const action of ['retry', 'dismiss']) {
|
||||
const response = await POST({ params: { id: row.id }, request: request({ action }) });
|
||||
expect(response.status).toBe(409);
|
||||
expect(await response.json()).toMatchObject({ error: expect.stringContaining('Provisioning is still active') });
|
||||
}
|
||||
expect(operations()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('keeps legacy drift previews readable and directs mutations to Watchdog without changing files', async () => {
|
||||
const { GET, POST } = await import('../../src/routes/api/drift/+server.js');
|
||||
const path = join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc');
|
||||
await writeFile(path, 'unexpected content');
|
||||
expect(await (await GET()).json()).toMatchObject({ totalDrift: 1, servers: [{ server, files: [expect.objectContaining({ relPath: 'stray.gsc' })] }] });
|
||||
const preview = await POST({ request: request({ dryRun: true, servers: [server] }) });
|
||||
expect(preview.status).toBe(200);
|
||||
const body = await preview.json();
|
||||
expect(body).toMatchObject({ ok: true, plan: { servers: [server], fingerprint: expect.any(String), files: [expect.objectContaining({ relPath: 'stray.gsc' })] } });
|
||||
const mutation = await POST({ request: request({ servers: [server], expectedFingerprint: body.plan.fingerprint }) });
|
||||
expect(mutation.status).toBe(410);
|
||||
expect(await mutation.json()).toMatchObject({ ok: false, watchdogUrl: '/watchdog', error: expect.stringContaining('Watchdog') });
|
||||
expect(await readFile(path, 'utf8')).toBe('unexpected content');
|
||||
expect(operations()).toEqual([]);
|
||||
expect(database.db().prepare('SELECT id FROM deployments').all()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Watchdog drift campaigns', () => {
|
||||
it('queues an audited rebuild without changing files until provisioning executes and retains its campaign through cleanup', async () => {
|
||||
await mkdir(join(fixture.t5, server, 'configs'), { recursive: true });
|
||||
const path = join(fixture.t5, server, 'configs', 'stray.cfg');
|
||||
await writeFile(path, 'unexpected content');
|
||||
await watchdog.watchdogTick(true);
|
||||
const row = onlyIncident();
|
||||
expect(row).toMatchObject({ kind: 'drift', state: 'recovering' });
|
||||
const action = watchdog.watchdogDetail(row.id)!.actions[0];
|
||||
expect(action).toMatchObject({ action: 'rebuild', campaign_id: expect.any(String), operation_state: 'waiting' });
|
||||
expect(await readFile(path, 'utf8')).toBe('unexpected content');
|
||||
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(database.db().prepare('SELECT rel_path, content_before, content_after FROM deployment_files WHERE deployment_id = ?').all(action.campaign_id))
|
||||
.toEqual([{ rel_path: 'stray.cfg', content_before: Buffer.from('unexpected content'), content_after: null }]);
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
const { processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
down();
|
||||
await processProvisioning();
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'queued' })]);
|
||||
await expect(access(path)).rejects.toThrow();
|
||||
expect(database.db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
|
||||
database.closeDb();
|
||||
await sweep(); await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
|
||||
expect(database.db().prepare('SELECT provisioning_id, restart_state FROM deployment_servers WHERE deployment_id = ?').get(action.campaign_id))
|
||||
.toEqual({ provisioning_id: action.operation_id, restart_state: 'pending' });
|
||||
await processCreations();
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
await processCreations(); await sweep(); await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(action.campaign_id)).toEqual({ restart_state: 'done' });
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'rebuild', state: 'done' })]);
|
||||
expect(komodo.komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps a missing rebuild key visible as attention without deleting files or retrying automatically', async () => {
|
||||
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
removeServerKey(server);
|
||||
const path = join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc');
|
||||
await writeFile(path, 'unexpected');
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(onlyIncident()).toMatchObject({ kind: 'drift', state: 'attention', summary: expect.stringContaining('Save a server key') });
|
||||
advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
|
||||
expect(operations()).toEqual([]);
|
||||
expect(await readFile(path, 'utf8')).toBe('unexpected');
|
||||
expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user