import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { prepareNativeHost } from '../helpers/native-host.js'; vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); const server = 't5_zombie_theater_1'; const campaign = 'recover-campaign'; const started = '2026-10-19T18:38:00.000Z'; const failedAt = '2026-10-19T18:48:42.000Z'; const recoveredAt = '2026-10-19T21:31:00.000Z'; let fixture: Fixture; beforeEach(async () => { vi.resetModules(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(started); fixture = await newFixture([server]); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); await prepareNativeHost([server]); }); afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); }); async function failedCampaign() { const { db } = await import('../../src/lib/server/db/index.js'); const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); const { finishDeployment } = await import('../../src/lib/server/provisioning/history.js'); const handle = db(); handle.prepare('UPDATE native_instances SET generation=5').run(); handle .prepare("INSERT INTO deployments(id,action,started_at) VALUES (?,'promote',?)") .run(campaign, started); handle .prepare( "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES (?,?,2,'failed')" ) .run(campaign, server); for (const path of ['common.gsc', 'leaderboard.gsc']) { queueFileChange({ sourceKind: 'campaign', sourceId: campaign, server, kind: 'script', relPath: path, content: Buffer.from(`intended ${path}`), inventorySource: campaign }); } const failed = requestRestart(server, { kind: 'campaign', id: campaign }); vi.setSystemTime(failedAt); finishDeployment(failed.id, { ok: false, message: 'Artifact download timed out' }); return { handle, failed }; } async function recoveredCampaign() { const { handle, failed } = await failedCampaign(); const { beginDeployment, finishDeployment } = await import('../../src/lib/server/provisioning/history.js'); const { captureRecipePlan, recipeStorage } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { recordScript } = await import('../../src/lib/server/fleet/inventory.js'); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); vi.setSystemTime('2026-10-19T21:29:00.000Z'); const recovery = beginDeployment(server, 'rebuild'); const instance = nativeInstance(server)!; const files = handle .prepare('SELECT kind,rel_path,content_hash FROM staged_file_changes WHERE source_id=?') .all(campaign) as { kind: 'script'; rel_path: string; content_hash: string }[]; const recipe = captureRecipePlan('t5', 'theater'); for (const file of files) { recipe.files.push({ kind: file.kind, path: file.rel_path, sha256: file.content_hash, name: file.rel_path, size: 10, versionId: null, version: null, releaseVersion: null }); recordScript(server, file.rel_path, null, file.content_hash, 10, 'native-create'); } const stored = recipeStorage(recipe); handle .prepare( 'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES (?,?,?)' ) .run(recovery, stored.recipe, stored.recipeSummary); handle .prepare( `INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,payload,state,created_at,updated_at) VALUES ('replacement',?,'instance.rebuild',?,5,'{"nextGeneration":6}','completed',?,?)` ) .run(instance.host_id, instance.id, new Date().toISOString(), recoveredAt); handle .prepare( 'UPDATE native_provisioning SET operation_id=?,applied_at=?,verified_at=? WHERE deployment_id=?' ) .run('replacement', recoveredAt, recoveredAt, recovery); handle.prepare('UPDATE native_instances SET generation=6 WHERE id=?').run(instance.id); handle .prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?') .run(new Date().toISOString(), recovery); vi.setSystemTime(recoveredAt); finishDeployment(recovery, { ok: true }); return { handle, failed, recovery, recipe, instance }; } async function sweep() { return (await import('../../src/lib/server/pipeline/sweep.js')).sweep(); } describe('campaign recovery', () => { it('adopts a verified matching rebuild, completes the campaign, and retains both operations and failure history', async () => { const { handle, failed, recovery } = await recoveredCampaign(); const original = handle .prepare('SELECT * FROM server_deployments WHERE id=?') .get(failed.id); const events = handle .prepare('SELECT * FROM server_deployment_events WHERE deployment_id=?') .all(failed.id); const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js'); expect(fleetOperations().find((op) => op.kind === 'campaign')).toMatchObject({ state: 'attention', evidenceAt: failedAt }); expect(await sweep()).toMatchObject({ healthy: 1, failed: 0 }); expect( handle .prepare( 'SELECT restart_state,provisioning_id,error,checked_at,restarted_at FROM deployment_servers' ) .get() ).toEqual({ restart_state: 'done', provisioning_id: recovery, error: null, checked_at: recoveredAt, restarted_at: '2026-10-19T21:29:00.000Z' }); expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({ finished_at: recoveredAt }); expect(fleetOperations()).toEqual([]); expect(serverOperations(server).find((op) => op.kind === 'campaign')).toMatchObject({ state: 'done', evidenceAt: recoveredAt }); expect( handle.prepare('SELECT * FROM server_deployments WHERE id=?').get(failed.id) ).toEqual(original); expect( handle .prepare('SELECT * FROM server_deployment_events WHERE deployment_id=?') .all(failed.id) ).toEqual(events); expect( handle .prepare( 'SELECT operation_id FROM provisioning_sources WHERE source_id=? ORDER BY rowid' ) .all(campaign) ).toEqual([{ operation_id: failed.id }, { operation_id: recovery }]); expect(handle.prepare('SELECT state FROM staged_file_changes').all()).toEqual([ { state: 'applied' }, { state: 'applied' } ]); const { GET } = await import('../../src/routes/api/campaigns/[id]/+server.js'); expect((await (await GET({ params: { id: campaign } })).json()).operations).toEqual([ expect.objectContaining({ id: failed.id, state: 'failed', finished_at: failedAt }), expect.objectContaining({ id: recovery, state: 'done', finished_at: recoveredAt }) ]); await sweep(); expect( handle .prepare("SELECT * FROM campaign_events WHERE message LIKE '%recovered by%'") .all() ).toHaveLength(1); }); it.each([ 'recipe mismatch', 'inventory mismatch', 'missing recipe', 'missing file', 'stale generation', 'wrong instance', 'wrong host', 'unverified', 'failed receipt', 'unrelated restart', 'older rebuild', 'cancelled files', 'deleted target', 'staged override mismatch' ])('keeps attention for %s', async (condition) => { const { handle, failed, recovery, recipe } = await recoveredCampaign(); if (condition === 'recipe mismatch') { recipe.files.find((f) => f.path === 'leaderboard.gsc')!.sha256 = 'other'; handle .prepare('UPDATE server_launch_recipes SET recipe_summary=?') .run(JSON.stringify(recipe)); } if (condition === 'inventory mismatch') handle .prepare( "UPDATE server_scripts SET sha256='other' WHERE rel_path='leaderboard.gsc'" ) .run(); if (condition === 'missing recipe') handle.prepare('DELETE FROM server_launch_recipes').run(); if (condition === 'missing file') handle .prepare("DELETE FROM staged_file_changes WHERE rel_path='leaderboard.gsc'") .run(); if (condition === 'stale generation') handle.prepare('UPDATE native_instances SET generation=7').run(); if (condition === 'wrong instance') handle.prepare("UPDATE native_instances SET server='replacement-name'").run(); if (condition === 'wrong host') handle.prepare("UPDATE native_instances SET host_id='other-host'").run(); if (condition === 'unverified') handle .prepare('UPDATE native_provisioning SET verified_at=NULL WHERE deployment_id=?') .run(recovery); if (condition === 'failed receipt') handle.prepare("UPDATE ec_host_operations SET state='failed'").run(); if (condition === 'unrelated restart') { handle .prepare("UPDATE server_deployments SET action='restart' WHERE id=?") .run(recovery); handle.prepare("UPDATE ec_host_operations SET kind='instance.restart'").run(); handle.prepare('UPDATE native_instances SET generation=5').run(); } if (condition === 'older rebuild') handle .prepare('UPDATE server_deployments SET started_at=? WHERE id=?') .run(started, recovery); if (condition === 'cancelled files') handle.prepare("UPDATE staged_file_changes SET state='cancelled'").run(); if (condition === 'deleted target') handle.prepare('UPDATE deployment_servers SET server_deleted_at=?').run(recoveredAt); if (condition === 'staged override mismatch') handle.prepare('UPDATE native_provisioning SET files_json=? WHERE deployment_id=?').run( JSON.stringify([ { path: 'runtime/plutonium/storage/t5/raw/scripts/sp/leaderboard.gsc', sha256: 'other' } ]), recovery ); await sweep(); expect( handle.prepare('SELECT restart_state,provisioning_id FROM deployment_servers').get() ).toEqual({ restart_state: 'failed', provisioning_id: failed.id }); expect( handle .prepare("SELECT * FROM campaign_events WHERE message LIKE '%recovered by%'") .all() ).toEqual([]); if (condition !== 'deleted target') { expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({ finished_at: null }); const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js'); expect(fleetOperations().find((op) => op.kind === 'campaign')).toMatchObject({ state: 'attention', evidenceAt: failedAt }); } }); it('does not accept fresh healthy status without a successful replacement operation', async () => { const { handle, failed } = await failedCampaign(); handle .prepare( "UPDATE servers SET last_seen_at=?,last_checked_at=?,last_error=NULL,container_state='running'" ) .run(recoveredAt, recoveredAt); await sweep(); expect( handle.prepare('SELECT restart_state,provisioning_id FROM deployment_servers').get() ).toEqual({ restart_state: 'failed', provisioning_id: failed.id }); const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js'); expect( fleetOperations() .filter((op) => op.state === 'attention') .map((op) => op.evidenceAt) ).toEqual([failedAt, failedAt]); }); it('waits for every remaining target before completing the campaign', async () => { const { handle } = await recoveredCampaign(); handle .prepare( "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES (?,'other',1,'held')" ) .run(campaign); await sweep(); expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({ finished_at: null }); handle .prepare("UPDATE deployment_servers SET restart_state='done' WHERE server='other'") .run(); await sweep(); expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({ finished_at: recoveredAt }); }); });