diff --git a/src/lib/fleet-overview.ts b/src/lib/fleet-overview.ts index 9dc1254..72aa05a 100644 --- a/src/lib/fleet-overview.ts +++ b/src/lib/fleet-overview.ts @@ -51,6 +51,7 @@ export interface FleetOperation { href: string; state: 'active' | 'attention' | 'done'; startedAt: string; + evidenceAt?: string | null; activityLabel?: string; resultState?: string; } diff --git a/src/lib/server/fleet/overview.ts b/src/lib/server/fleet/overview.ts index d4bd7f8..18fe42d 100644 --- a/src/lib/server/fleet/overview.ts +++ b/src/lib/server/fleet/overview.ts @@ -20,16 +20,23 @@ function readOperations(now: number, server?: string): FleetOperation[] { const operations: FleetOperation[] = []; const slots = handle .prepare( - `SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}` + `SELECT server, requested_at, checked_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}` ) - .all(bindings) as { server: string; requested_at: string; error: string | null }[]; + .all(bindings) as { + server: string; + requested_at: string; + checked_at: string | null; + error: string | null; + }[]; const recovering = new Map(slots.map((slot) => [slot.server, slot])); const representedRecovery = new Set(); const campaigns = handle .prepare( `SELECT d.id, d.action, d.started_at, ds.server, - ds.restart_state, ds.error, ds.restarted_at, ds.players + ds.restart_state, ds.error, ds.restarted_at, ds.players, + COALESCE(op.finished_at,op.observed_at,op.checked_at,ds.checked_at,d.started_at) AS evidence_at FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id + LEFT JOIN server_deployments op ON op.id=ds.provisioning_id WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done' ${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})` ) @@ -37,6 +44,7 @@ function readOperations(now: number, server?: string): FleetOperation[] { id: string; action: string; started_at: string; + evidence_at: string; server: string; restart_state: string; error: string | null; @@ -72,6 +80,7 @@ function readOperations(now: number, server?: string): FleetOperation[] { label, state, startedAt: row.started_at, + evidenceAt: row.evidence_at, detail: state === 'done' ? 'Update applied and restart checks completed' @@ -89,18 +98,21 @@ function readOperations(now: number, server?: string): FleetOperation[] { // index for detail history. Completed work is never loaded for the full fleet. const lifecycle = handle .prepare( - `SELECT id, server, action, state, stage, error, started_at FROM server_deployments + `SELECT id, server, action, state, stage, error, started_at, + COALESCE(finished_at,observed_at,checked_at,started_at) AS evidence_at FROM server_deployments WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health') ${ server === undefined ? '' : `UNION - SELECT id, server, action, state, stage, error, started_at FROM server_deployments + SELECT id, server, action, state, stage, error, started_at, + COALESCE(finished_at,observed_at,checked_at,started_at) AS evidence_at FROM server_deployments WHERE server_deleted_at IS NULL AND server = @server AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since` } UNION - SELECT id, server, action, state, stage, error, started_at FROM server_deployments + SELECT id, server, action, state, stage, error, started_at, + COALESCE(finished_at,observed_at,checked_at,started_at) AS evidence_at FROM server_deployments WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted') AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)` ) @@ -112,6 +124,7 @@ function readOperations(now: number, server?: string): FleetOperation[] { stage: string; error: string | null; started_at: string; + evidence_at: string; }[]; for (const row of lifecycle) { if (row.state === 'awaiting-health') representedRecovery.add(row.server); @@ -127,6 +140,7 @@ function readOperations(now: number, server?: string): FleetOperation[] { server: row.server, state, startedAt: row.started_at, + evidenceAt: row.evidence_at, activityLabel: `${actionLabels[row.action as ProvisioningAction] ?? row.action} instance`, resultState: row.state, label: @@ -145,12 +159,13 @@ function readOperations(now: number, server?: string): FleetOperation[] { } const betas = handle .prepare( - `SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error + `SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error, bs.updated_at FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')` ) .all(bindings) as { id: number; opened_at: string; + updated_at: string; server: string; stage: string; error: string | null; @@ -174,6 +189,7 @@ function readOperations(now: number, server?: string): FleetOperation[] { kind: 'beta', server: row.server, startedAt: row.opened_at, + evidenceAt: row.updated_at, state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage, detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, @@ -191,6 +207,7 @@ function readOperations(now: number, server?: string): FleetOperation[] { kind: string; detail: string | null; requested_at: string; + finished_at: string | null; }[]; for (const job of maintenance) { operations.push({ @@ -200,6 +217,7 @@ function readOperations(now: number, server?: string): FleetOperation[] { server: job.server, state: 'attention', startedAt: job.requested_at, + evidenceAt: job.finished_at ?? job.requested_at, label: job.kind === 'update' ? 'Update verification needs attention' @@ -216,6 +234,7 @@ function readOperations(now: number, server?: string): FleetOperation[] { kind: 'recovery', server: slot.server, startedAt: slot.requested_at, + evidenceAt: slot.checked_at ?? slot.requested_at, state: slot.error ? 'attention' : 'active', label: 'Recovering from restart', detail: slot.error ?? 'Verifying server health', diff --git a/src/lib/server/pipeline/campaign-recovery.ts b/src/lib/server/pipeline/campaign-recovery.ts new file mode 100644 index 0000000..c34f5b4 --- /dev/null +++ b/src/lib/server/pipeline/campaign-recovery.ts @@ -0,0 +1,125 @@ +import { posix } from 'node:path'; +import { db } from '../db/index.js'; +import { relDestForKind, type ArtifactKind } from '../config.js'; +import { recipeSummaryForDeployment } from '../fleet/launch-recipes.js'; +import { getServerScript } from '../fleet/inventory.js'; +import { recordProgress } from '../provisioning/progress.js'; +import type { LifecycleOperation } from '../provisioning/requests.js'; + +/** Adopt proven replacement files, never health alone or an unrelated restart. */ +export function reconcileCampaignRecovery( + campaign: string, + failed: LifecycleOperation, + filesChanged: number +): LifecycleOperation | null { + if (!['failed', 'interrupted', 'cancelled'].includes(failed.state)) return null; + return db().transaction(() => { + const files = db() + .prepare( + `SELECT kind, rel_path, content_hash, state FROM staged_file_changes + WHERE source_kind='campaign' AND source_id=? AND server=?` + ) + .all(campaign, failed.server) as { + kind: ArtifactKind; + rel_path: string; + content_hash: string | null; + state: string; + }[]; + // Partial staging and legacy campaigns without exact desired hashes need review. + if ( + !files.length || + files.length !== filesChanged || + files.some((f) => f.state === 'cancelled') + ) + return null; + const candidates = db() + .prepare( + `SELECT d.*, n.verified_at, n.files_json, i.generation AS current_generation + FROM server_deployments d + JOIN native_provisioning n ON n.deployment_id=d.id + JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL + JOIN ec_host_operations o ON o.id=n.operation_id AND o.instance_id=i.id AND o.host_id=i.host_id + WHERE d.server=? AND d.server_deleted_at IS NULL AND d.action='rebuild' AND d.state='done' + AND d.rowid > (SELECT rowid FROM server_deployments WHERE id=?) + AND d.started_at >= ? AND d.finished_at IS NOT NULL + AND n.applied_at IS NOT NULL AND n.verified_at IS NOT NULL + AND o.kind='instance.rebuild' AND o.state='completed' AND o.error IS NULL + AND n.generation=o.generation AND i.generation=o.generation+1 + ORDER BY d.rowid DESC` + ) + .all( + failed.server, + failed.id, + failed.finished_at ?? failed.started_at + ) as (LifecycleOperation & { + verified_at: string; + files_json: string | null; + current_generation: number; + })[]; + for (const candidate of candidates) { + const recipe = recipeSummaryForDeployment(candidate.id)?.recipe; + if (!recipe) continue; + // A campaign rebuild can override the promoted recipe with staged files. + const overrides = JSON.parse(candidate.files_json ?? '[]') as { + path: string; + sha256: string | null; + }[]; + const matches = files.every((file) => { + const override = overrides.find( + (entry) => + entry.path === + posix.join(relDestForKind(file.kind, recipe.game), file.rel_path) + ); + const installed = recipe.files.find( + (entry) => entry.kind === file.kind && entry.path === file.rel_path + ); + const hash = override ? override.sha256 : (installed?.sha256 ?? null); + return ( + hash === file.content_hash && + (getServerScript(failed.server, file.rel_path)?.sha256 ?? null) === + file.content_hash + ); + }); + if (!matches) continue; + const changed = db() + .prepare( + `UPDATE deployment_servers SET provisioning_id=?, restarted_at=? + WHERE deployment_id=? AND server=? AND server_deleted_at IS NULL + AND provisioning_id=? AND restart_state IN ('pending','failed')` + ) + .run(candidate.id, candidate.submitted_at, campaign, failed.server, failed.id); + if (!changed.changes) return null; + for (const id of [failed.id, candidate.id]) + db() + .prepare( + `INSERT OR IGNORE INTO provisioning_sources(operation_id,kind,source_id,server) + VALUES (?,'campaign',?,?)` + ) + .run(id, campaign, failed.server); + // Recovery proves the final bytes, but cannot invent an old rollback snapshot. + db() + .prepare( + `UPDATE staged_file_changes SET state='applied',applied_at=COALESCE(applied_at,?) + WHERE source_kind='campaign' AND source_id=? AND server=? AND state IN ('pending','applying')` + ) + .run(candidate.verified_at, campaign, failed.server); + const message = `${failed.server}: recovered by rebuild ${candidate.id} at ${candidate.verified_at} (generation ${candidate.current_generation}); original attempt ${failed.id} retained.`; + db() + .prepare('INSERT INTO campaign_events(deployment_id,message) VALUES (?,?)') + .run(campaign, message); + recordProgress(candidate.id, { + event: { + kind: 'campaign-recovered', + message, + details: { + campaignId: campaign, + failedOperationId: failed.id, + generation: candidate.current_generation + } + } + }); + return { ...candidate, checked_at: candidate.verified_at }; + } + return null; + })(); +} diff --git a/src/lib/server/pipeline/sweep.ts b/src/lib/server/pipeline/sweep.ts index da107b0..0c05c4e 100644 --- a/src/lib/server/pipeline/sweep.ts +++ b/src/lib/server/pipeline/sweep.ts @@ -1,6 +1,7 @@ import { db } from '../db/index.js'; import { emit } from './bus.js'; import { requestRestart } from '../provisioning/index.js'; +import { reconcileCampaignRecovery } from './campaign-recovery.js'; interface PendingRow { deployment_id: string; @@ -40,7 +41,9 @@ async function runSweep(): Promise { const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 }; for (const row of pending) { if (!row.files_changed) continue; // A failed file write is not a restart request. - const operation = requestRestart(row.server, { kind: 'campaign', id: row.deployment_id }); + const requested = requestRestart(row.server, { kind: 'campaign', id: row.deployment_id }); + const operation = + reconcileCampaignRecovery(row.deployment_id, requested, row.files_changed) ?? requested; result.checked++; const state = operation.state === 'done' diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 91439be..3c2519f 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -391,6 +391,9 @@ (item) => item.server === server.name )} {@const operationIssue = operation && status.state === 'online'} + {@const evidenceAt = operationIssue + ? (operation.evidenceAt ?? operation.startedAt) + : status.since} {@const actionLabel = operationIssue ? 'Review operation' : status.actionLabel}
@@ -412,9 +415,9 @@ >{operationIssue ? operation.label : status.reason}{operationIssue ? operation.label : status.shortReason}{#if status.since}{#if evidenceAt}{' · '}Evidence {elapsedSince( - status.since, + evidenceAt, $uiNow )} ago{/if} diff --git a/src/routes/api/campaigns/[id]/+server.ts b/src/routes/api/campaigns/[id]/+server.ts index 35ac35d..231ef04 100644 --- a/src/routes/api/campaigns/[id]/+server.ts +++ b/src/routes/api/campaigns/[id]/+server.ts @@ -10,6 +10,13 @@ export async function GET({ params }: { params: { id: string } }) { const servers = db() .prepare('SELECT * FROM deployment_servers WHERE deployment_id = ? ORDER BY server') .all(params.id); + const operations = db() + .prepare( + `SELECT d.id,d.server,d.action,d.state,d.finished_at FROM provisioning_sources s + JOIN server_deployments d ON d.id=s.operation_id + WHERE s.kind='campaign' AND s.source_id=? ORDER BY d.started_at,d.rowid` + ) + .all(params.id); // Don't ship blobs in the summary — the client asks for a single file when // it opens the diff viewer. @@ -56,6 +63,7 @@ export async function GET({ params }: { params: { id: string } }) { fileContentsExpired, deployment: dep, servers, + operations, files, scripts, events, diff --git a/src/routes/api/campaigns/[id]/resume/+server.ts b/src/routes/api/campaigns/[id]/resume/+server.ts index fddaba0..6d69ab3 100644 --- a/src/routes/api/campaigns/[id]/resume/+server.ts +++ b/src/routes/api/campaigns/[id]/resume/+server.ts @@ -9,18 +9,42 @@ export function POST({ params }: { params: { id: string } }) { .prepare( `SELECT ds.server, d.state FROM deployment_servers ds LEFT JOIN server_deployments d ON d.id = ds.provisioning_id - WHERE ds.deployment_id = ? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('held','manual')` + WHERE ds.deployment_id = ? AND ds.server_deleted_at IS NULL + AND (ds.restart_state IN ('held','manual') OR (ds.restart_state='failed' AND ds.files_changed>0))` ) .all(params.id) as { server: string; state: string | null }[]; for (const row of rows) { if (row.state && ['failed', 'interrupted', 'cancelled'].includes(row.state)) { releaseRestartRequest(row.server, { kind: 'campaign', id: params.id }); + // A failed health check may follow successful file application. A newer + // generation can have different files, so retry the retained desired bytes. + // Keep the original rollback image and refresh an existing compare guard. + db() + .prepare( + `UPDATE staged_file_changes SET state='pending',applied_at=NULL, + preserve_before_image=1,expected_before_sha=CASE WHEN check_before=1 THEN content_hash ELSE expected_before_sha END + WHERE source_kind='campaign' AND source_id=? AND server=? AND state='applied'` + ) + .run(params.id, row.server); + db() + .prepare( + `UPDATE deployment_servers SET restarted_at=NULL,checked_at=NULL,players=NULL + WHERE deployment_id=? AND server=?` + ) + .run(params.id, row.server); + db() + .prepare('INSERT INTO campaign_events(deployment_id,message) VALUES (?,?)') + .run( + params.id, + `${row.server}: retry requested; previous operation retained in history.` + ); } } const result = db() .prepare( `UPDATE deployment_servers SET restart_state = 'pending', error = NULL - WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('held','manual')` + WHERE deployment_id = ? AND server_deleted_at IS NULL + AND (restart_state IN ('held','manual') OR (restart_state='failed' AND files_changed>0))` ) .run(params.id); if (result.changes) @@ -28,7 +52,10 @@ export function POST({ params }: { params: { id: string } }) { return result.changes; })(); if (!changed) - return json({ error: 'This campaign has no held changes to resume.' }, { status: 409 }); + return json( + { error: 'This campaign has no held or failed activations to retry.' }, + { status: 409 } + ); void kick().catch((err) => console.error('[campaign] activation kick failed:', err)); return json({ ok: true }, { status: 202 }); } diff --git a/src/routes/campaigns/[id]/+page.svelte b/src/routes/campaigns/[id]/+page.svelte index 2271c11..9a67ebe 100644 --- a/src/routes/campaigns/[id]/+page.svelte +++ b/src/routes/campaigns/[id]/+page.svelte @@ -63,6 +63,13 @@ rolled_back_by: string | null; }; servers: ServerRow[]; + operations: { + id: string; + server: string; + action: string; + state: string; + finished_at: string | null; + }[]; files: FileRow[]; events: ActivityEvent[]; watchdogIncidentId?: string | null; @@ -386,9 +393,16 @@ >All servers activated{/if}
- {#if writable && detail.servers.some((server) => !server.server_deleted_at && ['held', 'manual'].includes(server.restart_state))} + {#if writable && detail.servers.some((server) => !server.server_deleted_at && (['held', 'manual'].includes(server.restart_state) || (server.restart_state === 'failed' && server.files_changed > 0)))} {detail.servers.some( + (server) => + !server.server_deleted_at && + server.restart_state === 'failed' && + server.files_changed > 0 + ) + ? 'Retry when empty' + : 'Resume when empty'} {/if}
@@ -525,6 +539,20 @@ $page.url )}>View provisioning{/if} + {#each detail.operations.filter((operation) => operation.server === s.server && operation.id !== s.provisioning_id) as operation (operation.id)} + + Previous {operation.action}: {operation.state} + {#if operation.finished_at} + · {/if} + + {/each} diff --git a/tests/unit/campaign-recovery.test.ts b/tests/unit/campaign-recovery.test.ts new file mode 100644 index 0000000..ef363a8 --- /dev/null +++ b/tests/unit/campaign-recovery.test.ts @@ -0,0 +1,318 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import { prepareNativeHost } from '../helpers/native-host.js'; + +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); + +const server = 't5_zombie_theater_1'; +const campaign = 'recover-campaign'; +const started = '2026-10-19T18:38:00.000Z'; +const failedAt = '2026-10-19T18:48:42.000Z'; +const recoveredAt = '2026-10-19T21:31:00.000Z'; +let fixture: Fixture; + +beforeEach(async () => { + vi.resetModules(); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(started); + fixture = await newFixture([server]); + const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); + await seedServerRegistry(); + await prepareNativeHost([server]); +}); +afterEach(async () => { + await fixture.cleanup(); + vi.useRealTimers(); +}); + +async function failedCampaign() { + const { db } = await import('../../src/lib/server/db/index.js'); + const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); + const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); + const { finishDeployment } = await import('../../src/lib/server/provisioning/history.js'); + const handle = db(); + handle.prepare('UPDATE native_instances SET generation=5').run(); + handle + .prepare("INSERT INTO deployments(id,action,started_at) VALUES (?,'promote',?)") + .run(campaign, started); + handle + .prepare( + "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES (?,?,2,'failed')" + ) + .run(campaign, server); + for (const path of ['common.gsc', 'leaderboard.gsc']) { + queueFileChange({ + sourceKind: 'campaign', + sourceId: campaign, + server, + kind: 'script', + relPath: path, + content: Buffer.from(`intended ${path}`), + inventorySource: campaign + }); + } + const failed = requestRestart(server, { kind: 'campaign', id: campaign }); + vi.setSystemTime(failedAt); + finishDeployment(failed.id, { ok: false, message: 'Artifact download timed out' }); + return { handle, failed }; +} + +async function recoveredCampaign() { + const { handle, failed } = await failedCampaign(); + const { beginDeployment, finishDeployment } = + await import('../../src/lib/server/provisioning/history.js'); + const { captureRecipePlan, recipeStorage } = + await import('../../src/lib/server/fleet/launch-recipes.js'); + const { recordScript } = await import('../../src/lib/server/fleet/inventory.js'); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + vi.setSystemTime('2026-10-19T21:29:00.000Z'); + const recovery = beginDeployment(server, 'rebuild'); + const instance = nativeInstance(server)!; + const files = handle + .prepare('SELECT kind,rel_path,content_hash FROM staged_file_changes WHERE source_id=?') + .all(campaign) as { kind: 'script'; rel_path: string; content_hash: string }[]; + const recipe = captureRecipePlan('t5', 'theater'); + for (const file of files) { + recipe.files.push({ + kind: file.kind, + path: file.rel_path, + sha256: file.content_hash, + name: file.rel_path, + size: 10, + versionId: null, + version: null, + releaseVersion: null + }); + recordScript(server, file.rel_path, null, file.content_hash, 10, 'native-create'); + } + const stored = recipeStorage(recipe); + handle + .prepare( + 'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES (?,?,?)' + ) + .run(recovery, stored.recipe, stored.recipeSummary); + handle + .prepare( + `INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,payload,state,created_at,updated_at) + VALUES ('replacement',?,'instance.rebuild',?,5,'{"nextGeneration":6}','completed',?,?)` + ) + .run(instance.host_id, instance.id, new Date().toISOString(), recoveredAt); + handle + .prepare( + 'UPDATE native_provisioning SET operation_id=?,applied_at=?,verified_at=? WHERE deployment_id=?' + ) + .run('replacement', recoveredAt, recoveredAt, recovery); + handle.prepare('UPDATE native_instances SET generation=6 WHERE id=?').run(instance.id); + handle + .prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?') + .run(new Date().toISOString(), recovery); + vi.setSystemTime(recoveredAt); + finishDeployment(recovery, { ok: true }); + return { handle, failed, recovery, recipe, instance }; +} + +async function sweep() { + return (await import('../../src/lib/server/pipeline/sweep.js')).sweep(); +} + +describe('campaign recovery', () => { + it('adopts a verified matching rebuild, completes the campaign, and retains both operations and failure history', async () => { + const { handle, failed, recovery } = await recoveredCampaign(); + const original = handle + .prepare('SELECT * FROM server_deployments WHERE id=?') + .get(failed.id); + const events = handle + .prepare('SELECT * FROM server_deployment_events WHERE deployment_id=?') + .all(failed.id); + const { fleetOperations, serverOperations } = + await import('../../src/lib/server/fleet/overview.js'); + expect(fleetOperations().find((op) => op.kind === 'campaign')).toMatchObject({ + state: 'attention', + evidenceAt: failedAt + }); + expect(await sweep()).toMatchObject({ healthy: 1, failed: 0 }); + expect( + handle + .prepare( + 'SELECT restart_state,provisioning_id,error,checked_at,restarted_at FROM deployment_servers' + ) + .get() + ).toEqual({ + restart_state: 'done', + provisioning_id: recovery, + error: null, + checked_at: recoveredAt, + restarted_at: '2026-10-19T21:29:00.000Z' + }); + expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({ + finished_at: recoveredAt + }); + expect(fleetOperations()).toEqual([]); + expect(serverOperations(server).find((op) => op.kind === 'campaign')).toMatchObject({ + state: 'done', + evidenceAt: recoveredAt + }); + expect( + handle.prepare('SELECT * FROM server_deployments WHERE id=?').get(failed.id) + ).toEqual(original); + expect( + handle + .prepare('SELECT * FROM server_deployment_events WHERE deployment_id=?') + .all(failed.id) + ).toEqual(events); + expect( + handle + .prepare( + 'SELECT operation_id FROM provisioning_sources WHERE source_id=? ORDER BY rowid' + ) + .all(campaign) + ).toEqual([{ operation_id: failed.id }, { operation_id: recovery }]); + expect(handle.prepare('SELECT state FROM staged_file_changes').all()).toEqual([ + { state: 'applied' }, + { state: 'applied' } + ]); + const { GET } = await import('../../src/routes/api/campaigns/[id]/+server.js'); + expect((await (await GET({ params: { id: campaign } })).json()).operations).toEqual([ + expect.objectContaining({ id: failed.id, state: 'failed', finished_at: failedAt }), + expect.objectContaining({ id: recovery, state: 'done', finished_at: recoveredAt }) + ]); + await sweep(); + expect( + handle + .prepare("SELECT * FROM campaign_events WHERE message LIKE '%recovered by%'") + .all() + ).toHaveLength(1); + }); + + it.each([ + 'recipe mismatch', + 'inventory mismatch', + 'missing recipe', + 'missing file', + 'stale generation', + 'wrong instance', + 'wrong host', + 'unverified', + 'failed receipt', + 'unrelated restart', + 'older rebuild', + 'cancelled files', + 'deleted target', + 'staged override mismatch' + ])('keeps attention for %s', async (condition) => { + const { handle, failed, recovery, recipe } = await recoveredCampaign(); + if (condition === 'recipe mismatch') { + recipe.files.find((f) => f.path === 'leaderboard.gsc')!.sha256 = 'other'; + handle + .prepare('UPDATE server_launch_recipes SET recipe_summary=?') + .run(JSON.stringify(recipe)); + } + if (condition === 'inventory mismatch') + handle + .prepare( + "UPDATE server_scripts SET sha256='other' WHERE rel_path='leaderboard.gsc'" + ) + .run(); + if (condition === 'missing recipe') + handle.prepare('DELETE FROM server_launch_recipes').run(); + if (condition === 'missing file') + handle + .prepare("DELETE FROM staged_file_changes WHERE rel_path='leaderboard.gsc'") + .run(); + if (condition === 'stale generation') + handle.prepare('UPDATE native_instances SET generation=7').run(); + if (condition === 'wrong instance') + handle.prepare("UPDATE native_instances SET server='replacement-name'").run(); + if (condition === 'wrong host') + handle.prepare("UPDATE native_instances SET host_id='other-host'").run(); + if (condition === 'unverified') + handle + .prepare('UPDATE native_provisioning SET verified_at=NULL WHERE deployment_id=?') + .run(recovery); + if (condition === 'failed receipt') + handle.prepare("UPDATE ec_host_operations SET state='failed'").run(); + if (condition === 'unrelated restart') { + handle + .prepare("UPDATE server_deployments SET action='restart' WHERE id=?") + .run(recovery); + handle.prepare("UPDATE ec_host_operations SET kind='instance.restart'").run(); + handle.prepare('UPDATE native_instances SET generation=5').run(); + } + if (condition === 'older rebuild') + handle + .prepare('UPDATE server_deployments SET started_at=? WHERE id=?') + .run(started, recovery); + if (condition === 'cancelled files') + handle.prepare("UPDATE staged_file_changes SET state='cancelled'").run(); + if (condition === 'deleted target') + handle.prepare('UPDATE deployment_servers SET server_deleted_at=?').run(recoveredAt); + if (condition === 'staged override mismatch') + handle.prepare('UPDATE native_provisioning SET files_json=? WHERE deployment_id=?').run( + JSON.stringify([ + { + path: 'runtime/plutonium/storage/t5/raw/scripts/sp/leaderboard.gsc', + sha256: 'other' + } + ]), + recovery + ); + await sweep(); + expect( + handle.prepare('SELECT restart_state,provisioning_id FROM deployment_servers').get() + ).toEqual({ restart_state: 'failed', provisioning_id: failed.id }); + expect( + handle + .prepare("SELECT * FROM campaign_events WHERE message LIKE '%recovered by%'") + .all() + ).toEqual([]); + if (condition !== 'deleted target') { + expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({ + finished_at: null + }); + const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js'); + expect(fleetOperations().find((op) => op.kind === 'campaign')).toMatchObject({ + state: 'attention', + evidenceAt: failedAt + }); + } + }); + + it('does not accept fresh healthy status without a successful replacement operation', async () => { + const { handle, failed } = await failedCampaign(); + handle + .prepare( + "UPDATE servers SET last_seen_at=?,last_checked_at=?,last_error=NULL,container_state='running'" + ) + .run(recoveredAt, recoveredAt); + await sweep(); + expect( + handle.prepare('SELECT restart_state,provisioning_id FROM deployment_servers').get() + ).toEqual({ restart_state: 'failed', provisioning_id: failed.id }); + const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js'); + expect( + fleetOperations() + .filter((op) => op.state === 'attention') + .map((op) => op.evidenceAt) + ).toEqual([failedAt, failedAt]); + }); + + it('waits for every remaining target before completing the campaign', async () => { + const { handle } = await recoveredCampaign(); + handle + .prepare( + "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES (?,'other',1,'held')" + ) + .run(campaign); + await sweep(); + expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({ + finished_at: null + }); + handle + .prepare("UPDATE deployment_servers SET restart_state='done' WHERE server='other'") + .run(); + await sweep(); + expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({ + finished_at: recoveredAt + }); + }); +}); diff --git a/tests/unit/campaign-resume.test.ts b/tests/unit/campaign-resume.test.ts index 92a2b15..ac0a9ca 100644 --- a/tests/unit/campaign-resume.test.ts +++ b/tests/unit/campaign-resume.test.ts @@ -57,7 +57,7 @@ async function resume(id = campaign) { } describe('campaign resume API', () => { - it.each(['held', 'manual'])( + it.each(['held', 'manual', 'failed'])( 'queues %s rollback changes without touching live files', async (state) => { await heldRollback(state); @@ -93,7 +93,7 @@ describe('campaign resume API', () => { it.each(['cancelled', 'failed', 'interrupted'])( 'releases an old %s operation so the worker creates fresh restart intent', async (state) => { - await heldRollback(); + await heldRollback('failed'); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); const { db } = await import('../../src/lib/server/db/index.js'); @@ -101,6 +101,11 @@ describe('campaign resume API', () => { db() .prepare('UPDATE server_deployments SET state = ?, finished_at = ? WHERE id = ?') .run(state, new Date().toISOString(), old.id); + db() + .prepare( + 'UPDATE deployment_servers SET restarted_at=?,checked_at=?,players=3 WHERE deployment_id=?' + ) + .run(new Date().toISOString(), new Date().toISOString(), campaign); expect((await resume()).status).toBe(202); expect( db() @@ -113,7 +118,14 @@ describe('campaign resume API', () => { await sweep(); const next = requestRestart(server, { kind: 'campaign', id: campaign }); expect(next.id).not.toBe(old.id); - expect(next).toMatchObject({ state: 'waiting', submitted_at: null }); + expect(next).toMatchObject({ state: 'waiting', action: 'rebuild', submitted_at: null }); + expect( + db() + .prepare( + 'SELECT restarted_at,checked_at,players FROM deployment_servers WHERE deployment_id=?' + ) + .get(campaign) + ).toEqual({ restarted_at: null, checked_at: null, players: null }); expect( db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.id) ).toEqual({ state }); @@ -174,7 +186,7 @@ describe('campaign resume API', () => { const response = await resume(); expect(response.status).toBe(409); expect(await response.json()).toEqual({ - error: 'This campaign has no held changes to resume.' + error: 'This campaign has no held or failed activations to retry.' }); expect(kick).not.toHaveBeenCalled(); expect(db().prepare('SELECT * FROM server_deployments').all()).toEqual([]); @@ -193,4 +205,11 @@ describe('campaign resume API', () => { } } ); + + it('does not retry a failed file-staging attempt with no activation work', async () => { + await heldRollback('failed'); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare('UPDATE deployment_servers SET files_changed=0').run(); + expect((await resume()).status).toBe(409); + }); }); diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index f8e5801..b320ac9 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -1948,6 +1948,145 @@ describe('native instance provisioning', () => { expect(nativeInstance(job.name)?.generation).toBe(3); }); + it.each(['pending', 'applied'])( + 'retries %s files from a failed campaign on the current generation through occupancy and startup admission', + async (fileState) => { + const job = await queue(); + await advance(); + await acknowledge(job.deploymentId); + const { db } = await import('../../src/lib/server/db/index.js'); + const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js'); + const { requestRestart } = + await import('../../src/lib/server/provisioning/requests.js'); + const { finishDeployment } = + await import('../../src/lib/server/provisioning/history.js'); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { processProvisioning } = + await import('../../src/lib/server/provisioning/worker.js'); + db() + .prepare( + "INSERT INTO deployments(id,action,started_at) VALUES('retry','promote',?)" + ) + .run(new Date().toISOString()); + db() + .prepare( + "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('retry',?,1,'failed')" + ) + .run(job.name); + queueFileChange({ + sourceKind: 'campaign', + sourceId: 'retry', + server: job.name, + kind: 'script', + relPath: 'retry.gsc', + content: Buffer.from('desired campaign bytes'), + inventorySource: 'test' + }); + const failed = requestRestart(job.name, { kind: 'campaign', id: 'retry' }); + finishDeployment(failed.id, { ok: false, message: 'Download timeout' }); + db() + .prepare( + "UPDATE staged_file_changes SET state=? WHERE source_kind='campaign' AND source_id='retry'" + ) + .run(fileState); + const { queueNativeRebuild } = + await import('../../src/lib/server/provisioning/native.js'); + const unrelated = queueNativeRebuild(job.name, 'ui'); + if (!unrelated.ok) throw new Error(unrelated.message); + await advance(); + await acknowledge(unrelated.deploymentId); + await sweep(); + expect( + db() + .prepare( + "SELECT restart_state FROM deployment_servers WHERE deployment_id='retry'" + ) + .get() + ).toEqual({ restart_state: 'failed' }); + + const { fleetSettings, saveFleetSettings } = + await import('../../src/lib/server/settings.js'); + saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 }); + const { queueCreateServer } = + await import('../../src/lib/server/provisioning/instances.js'); + const other = queueCreateServer({ + game: 't5', + gameMode: 'zombie', + mapSlug: 'theater', + hostId, + libraryId, + image, + plutoKey: 'second-key' + }); + if (!other.ok) throw new Error(other.message); + await advance(); + expect(await operationFor(other.deploymentId)).not.toBeNull(); + + db() + .prepare("UPDATE deployment_servers SET restarted_at=? WHERE deployment_id='retry'") + .run(new Date().toISOString()); + const { POST } = await import('../../src/routes/api/campaigns/[id]/resume/+server.js'); + expect(POST({ params: { id: 'retry' } }).status).toBe(202); + await sweep(); + const next = requestRestart(job.name, { kind: 'campaign', id: 'retry' }); + expect(next).toMatchObject({ action: 'rebuild', state: 'waiting', submitted_at: null }); + expect(next.id).not.toBe(failed.id); + expect( + db() + .prepare('SELECT generation FROM native_provisioning WHERE deployment_id=?') + .get(next.id) + ).toEqual({ generation: 2 }); + await processProvisioning(); + const staged = db() + .prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?') + .get(next.id) as { stage_operation_id: string }; + const { reportHostOperation, getHostOperation } = + await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' }); + await processProvisioning(); + expect(await operationFor(next.id)).toBeNull(); + expect((await detail(next.id)).stage).toContain('startup capacity'); + remote.players = 2; + await acknowledge(other.deploymentId); + await processProvisioning(); + expect(await operationFor(next.id)).toBeNull(); + expect((await detail(next.id)).stage).toContain('player'); + remote.players = 0; + await processProvisioning(); + const activation = (await operationFor(next.id))!; + expect(activation, (await detail(next.id)).stage).toMatchObject({ + kind: 'instance.rebuild', + generation: 2, + payload: { nextGeneration: 3, stageId: next.id } + }); + const files = getHostOperation(staged.stage_operation_id)!.payload.files as { + id: number; + path: string; + }[]; + reportHostOperation(hostId, activation.id, { + state: 'completed', + result: { before: files.map((file) => ({ ...file, contentBase64: null })) } + }); + await processProvisioning(); + await sweep(); + expect( + db() + .prepare( + "SELECT restart_state,provisioning_id FROM deployment_servers WHERE deployment_id='retry'" + ) + .get() + ).toEqual({ restart_state: 'done', provisioning_id: next.id }); + expect((await detail(failed.id)).state).toBe('failed'); + expect( + db() + .prepare( + "SELECT operation_id FROM provisioning_sources WHERE kind='campaign' AND source_id='retry' ORDER BY rowid" + ) + .all() + ).toEqual([{ operation_id: failed.id }, { operation_id: next.id }]); + } + ); + it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => { const job = await queue(); await advance(); diff --git a/tests/unit/provisioning-history-performance.test.ts b/tests/unit/provisioning-history-performance.test.ts index 6da088b..a7c9275 100644 --- a/tests/unit/provisioning-history-performance.test.ts +++ b/tests/unit/provisioning-history-performance.test.ts @@ -87,7 +87,7 @@ describe('provisioning reads with retained lifecycle history', () => { fleetOperations(now); const queries = prepare.mock.calls .map(([sql]) => sql) - .filter((sql) => sql.includes('server_deployments')); + .filter((sql) => sql.includes('server_deployments') && sql.includes('UNION')); prepare.mockRestore(); expect(queries).toHaveLength(1); const statement = handle.prepare(queries[0]); @@ -98,7 +98,8 @@ describe('provisioning reads with retained lifecycle history', () => { 'state', 'stage', 'error', - 'started_at' + 'started_at', + 'evidence_at' ]); const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all() as { detail: string; diff --git a/tests/unit/ui-fleet-attention.test.ts b/tests/unit/ui-fleet-attention.test.ts new file mode 100644 index 0000000..e6ff1f9 --- /dev/null +++ b/tests/unit/ui-fleet-attention.test.ts @@ -0,0 +1,90 @@ +import { expect, it, vi } from 'vitest'; +import { render } from 'svelte/server'; +import { writable } from 'svelte/store'; +import type { ComponentProps } from 'svelte'; +import Page from '../../src/routes/+page.svelte'; +import { syncServerTime } from '../../src/lib/ui/stores/servers.js'; +import type { FleetServer } from '../../src/lib/fleet-overview.js'; + +vi.mock('$app/navigation', () => ({ + afterNavigate: vi.fn(), + replaceState: vi.fn(), + goto: vi.fn(), + beforeNavigate: vi.fn() +})); +vi.mock('$app/stores', () => ({ + page: writable({ url: new URL('http://localhost/'), data: { user: { role: 'viewer' } } }) +})); + +const now = Date.parse('2026-10-19T22:46:00.000Z'); +const healthAt = new Date(now - 8000).toISOString(); +const failedAt = '2026-10-19T18:48:42.000Z'; +const server: FleetServer = { + name: 't5_zombie_sumpf_1', + game: 't5', + map: 'zombie_sumpf', + last_map: 'zombie_sumpf', + last_players: 0, + last_gametype: 'zom', + last_seen_at: healthAt, + last_checked_at: healthAt, + container_state: 'running', + container_checked_at: healthAt, + last_error: null, + roster: [], + health: 'online', + held: [], + active_since: null, + inactive_since: null, + game_started_at: null, + game_ended_at: null +}; + +it.each([true, false])( + 'dates the displayed attention reason when the server is healthy=%s', + (healthy) => { + syncServerTime(now); + const data = { + user: null, + setupRequired: false, + fleetSnapshot: { + serverNow: now, + hostAttention: 0, + reservedNames: [], + servers: [ + healthy ? server : { ...server, health: 'offline', container_state: 'stopped' } + ], + integrations: { + native: true, + hostsReady: 1, + hostsTotal: 1, + rcon: true, + rconTimeoutMs: 2000, + defaultRconHost: 'localhost' + }, + operations: [ + { + key: 'campaign:one', + group: 'campaign:one', + kind: 'campaign', + server: server.name, + label: 'Update needs attention', + detail: 'Download timed out', + href: '/campaigns/one', + state: 'attention', + startedAt: '2026-10-19T18:38:00.000Z', + evidenceAt: failedAt + } + ] + } + // Rendering needs the public Fleet fields, without the registry's storage metadata. + } as unknown as ComponentProps['data']; + const body = render(Page, { props: { data } }).body; + const evidence = body.match(/class="evidence-time[^"]*" title="([^"]+)"/)?.[1]; + expect(evidence).toBe(healthy ? failedAt : healthAt); + if (healthy) { + expect(body).toContain('Update needs attention'); + expect(body).toContain('Evidence 3h 57m ago'); + } + } +);