Reconcile recovered campaign targets (Refs #166)

src/lib/fleet-overview.ts

@@ -51,6 +51,7 @@ export interface FleetOperation {
     href: string;
     state: 'active' | 'attention' | 'done';
     startedAt: string;
+    evidenceAt?: string | null;
     activityLabel?: string;
     resultState?: string;
 }

src/lib/server/fleet/overview.ts

@@ -20,16 +20,23 @@ function readOperations(now: number, server?: string): FleetOperation[] {
     const operations: FleetOperation[] = [];
     const slots = handle
         .prepare(
-            `SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`
+            `SELECT server, requested_at, checked_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`
         )
-        .all(bindings) as { server: string; requested_at: string; error: string | null }[];
+        .all(bindings) as {
+        server: string;
+        requested_at: string;
+        checked_at: string | null;
+        error: string | null;
+    }[];
     const recovering = new Map(slots.map((slot) => [slot.server, slot]));
     const representedRecovery = new Set<string>();
     const campaigns = handle
         .prepare(
             `SELECT d.id, d.action, d.started_at, ds.server,
-        ds.restart_state, ds.error, ds.restarted_at, ds.players
+        ds.restart_state, ds.error, ds.restarted_at, ds.players,
+        COALESCE(op.finished_at,op.observed_at,op.checked_at,ds.checked_at,d.started_at) AS evidence_at
         FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
+        LEFT JOIN server_deployments op ON op.id=ds.provisioning_id
         WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done'
             ${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`
         )
@@ -37,6 +44,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
         id: string;
         action: string;
         started_at: string;
+        evidence_at: string;
         server: string;
         restart_state: string;
         error: string | null;
@@ -72,6 +80,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
             label,
             state,
             startedAt: row.started_at,
+            evidenceAt: row.evidence_at,
             detail:
                 state === 'done'
                     ? 'Update applied and restart checks completed'
@@ -89,18 +98,21 @@ function readOperations(now: number, server?: string): FleetOperation[] {
     // index for detail history. Completed work is never loaded for the full fleet.
     const lifecycle = handle
         .prepare(
-            `SELECT id, server, action, state, stage, error, started_at FROM server_deployments
+            `SELECT id, server, action, state, stage, error, started_at,
+                COALESCE(finished_at,observed_at,checked_at,started_at) AS evidence_at FROM server_deployments
         WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
         ${
             server === undefined
                 ? ''
                 : `UNION
-        SELECT id, server, action, state, stage, error, started_at FROM server_deployments
+        SELECT id, server, action, state, stage, error, started_at,
+            COALESCE(finished_at,observed_at,checked_at,started_at) AS evidence_at FROM server_deployments
         WHERE server_deleted_at IS NULL AND server = @server
             AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`
         }
         UNION
-        SELECT id, server, action, state, stage, error, started_at FROM server_deployments
+        SELECT id, server, action, state, stage, error, started_at,
+            COALESCE(finished_at,observed_at,checked_at,started_at) AS evidence_at FROM server_deployments
         WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted')
             AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`
         )
@@ -112,6 +124,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
         stage: string;
         error: string | null;
         started_at: string;
+        evidence_at: string;
     }[];
     for (const row of lifecycle) {
         if (row.state === 'awaiting-health') representedRecovery.add(row.server);
@@ -127,6 +140,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
             server: row.server,
             state,
             startedAt: row.started_at,
+            evidenceAt: row.evidence_at,
             activityLabel: `${actionLabels[row.action as ProvisioningAction] ?? row.action} instance`,
             resultState: row.state,
             label:
@@ -145,12 +159,13 @@ function readOperations(now: number, server?: string): FleetOperation[] {
     }
     const betas = handle
         .prepare(
-            `SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
+            `SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error, bs.updated_at
         FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`
         )
         .all(bindings) as {
         id: number;
         opened_at: string;
+        updated_at: string;
         server: string;
         stage: string;
         error: string | null;
@@ -174,6 +189,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
             kind: 'beta',
             server: row.server,
             startedAt: row.opened_at,
+            evidenceAt: row.updated_at,
             state: row.stage === 'failed' ? 'attention' : 'active',
             label: stages[row.stage] ?? row.stage,
             detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`,
@@ -191,6 +207,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
         kind: string;
         detail: string | null;
         requested_at: string;
+        finished_at: string | null;
     }[];
     for (const job of maintenance) {
         operations.push({
@@ -200,6 +217,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
             server: job.server,
             state: 'attention',
             startedAt: job.requested_at,
+            evidenceAt: job.finished_at ?? job.requested_at,
             label:
                 job.kind === 'update'
                     ? 'Update verification needs attention'
@@ -216,6 +234,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
             kind: 'recovery',
             server: slot.server,
             startedAt: slot.requested_at,
+            evidenceAt: slot.checked_at ?? slot.requested_at,
             state: slot.error ? 'attention' : 'active',
             label: 'Recovering from restart',
             detail: slot.error ?? 'Verifying server health',

src/lib/server/pipeline/campaign-recovery.ts

@@ -0,0 +1,125 @@
+import { posix } from 'node:path';
+import { db } from '../db/index.js';
+import { relDestForKind, type ArtifactKind } from '../config.js';
+import { recipeSummaryForDeployment } from '../fleet/launch-recipes.js';
+import { getServerScript } from '../fleet/inventory.js';
+import { recordProgress } from '../provisioning/progress.js';
+import type { LifecycleOperation } from '../provisioning/requests.js';
+
+/** Adopt proven replacement files, never health alone or an unrelated restart. */
+export function reconcileCampaignRecovery(
+    campaign: string,
+    failed: LifecycleOperation,
+    filesChanged: number
+): LifecycleOperation | null {
+    if (!['failed', 'interrupted', 'cancelled'].includes(failed.state)) return null;
+    return db().transaction(() => {
+        const files = db()
+            .prepare(
+                `SELECT kind, rel_path, content_hash, state FROM staged_file_changes
+                WHERE source_kind='campaign' AND source_id=? AND server=?`
+            )
+            .all(campaign, failed.server) as {
+            kind: ArtifactKind;
+            rel_path: string;
+            content_hash: string | null;
+            state: string;
+        }[];
+        // Partial staging and legacy campaigns without exact desired hashes need review.
+        if (
+            !files.length ||
+            files.length !== filesChanged ||
+            files.some((f) => f.state === 'cancelled')
+        )
+            return null;
+        const candidates = db()
+            .prepare(
+                `SELECT d.*, n.verified_at, n.files_json, i.generation AS current_generation
+                FROM server_deployments d
+                JOIN native_provisioning n ON n.deployment_id=d.id
+                JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL
+                JOIN ec_host_operations o ON o.id=n.operation_id AND o.instance_id=i.id AND o.host_id=i.host_id
+                WHERE d.server=? AND d.server_deleted_at IS NULL AND d.action='rebuild' AND d.state='done'
+                AND d.rowid > (SELECT rowid FROM server_deployments WHERE id=?)
+                AND d.started_at >= ? AND d.finished_at IS NOT NULL
+                AND n.applied_at IS NOT NULL AND n.verified_at IS NOT NULL
+                AND o.kind='instance.rebuild' AND o.state='completed' AND o.error IS NULL
+                AND n.generation=o.generation AND i.generation=o.generation+1
+                ORDER BY d.rowid DESC`
+            )
+            .all(
+                failed.server,
+                failed.id,
+                failed.finished_at ?? failed.started_at
+            ) as (LifecycleOperation & {
+            verified_at: string;
+            files_json: string | null;
+            current_generation: number;
+        })[];
+        for (const candidate of candidates) {
+            const recipe = recipeSummaryForDeployment(candidate.id)?.recipe;
+            if (!recipe) continue;
+            // A campaign rebuild can override the promoted recipe with staged files.
+            const overrides = JSON.parse(candidate.files_json ?? '[]') as {
+                path: string;
+                sha256: string | null;
+            }[];
+            const matches = files.every((file) => {
+                const override = overrides.find(
+                    (entry) =>
+                        entry.path ===
+                        posix.join(relDestForKind(file.kind, recipe.game), file.rel_path)
+                );
+                const installed = recipe.files.find(
+                    (entry) => entry.kind === file.kind && entry.path === file.rel_path
+                );
+                const hash = override ? override.sha256 : (installed?.sha256 ?? null);
+                return (
+                    hash === file.content_hash &&
+                    (getServerScript(failed.server, file.rel_path)?.sha256 ?? null) ===
+                        file.content_hash
+                );
+            });
+            if (!matches) continue;
+            const changed = db()
+                .prepare(
+                    `UPDATE deployment_servers SET provisioning_id=?, restarted_at=?
+                    WHERE deployment_id=? AND server=? AND server_deleted_at IS NULL
+                    AND provisioning_id=? AND restart_state IN ('pending','failed')`
+                )
+                .run(candidate.id, candidate.submitted_at, campaign, failed.server, failed.id);
+            if (!changed.changes) return null;
+            for (const id of [failed.id, candidate.id])
+                db()
+                    .prepare(
+                        `INSERT OR IGNORE INTO provisioning_sources(operation_id,kind,source_id,server)
+                        VALUES (?,'campaign',?,?)`
+                    )
+                    .run(id, campaign, failed.server);
+            // Recovery proves the final bytes, but cannot invent an old rollback snapshot.
+            db()
+                .prepare(
+                    `UPDATE staged_file_changes SET state='applied',applied_at=COALESCE(applied_at,?)
+                    WHERE source_kind='campaign' AND source_id=? AND server=? AND state IN ('pending','applying')`
+                )
+                .run(candidate.verified_at, campaign, failed.server);
+            const message = `${failed.server}: recovered by rebuild ${candidate.id} at ${candidate.verified_at} (generation ${candidate.current_generation}); original attempt ${failed.id} retained.`;
+            db()
+                .prepare('INSERT INTO campaign_events(deployment_id,message) VALUES (?,?)')
+                .run(campaign, message);
+            recordProgress(candidate.id, {
+                event: {
+                    kind: 'campaign-recovered',
+                    message,
+                    details: {
+                        campaignId: campaign,
+                        failedOperationId: failed.id,
+                        generation: candidate.current_generation
+                    }
+                }
+            });
+            return { ...candidate, checked_at: candidate.verified_at };
+        }
+        return null;
+    })();
+}

src/lib/server/pipeline/sweep.ts

@@ -1,6 +1,7 @@
 import { db } from '../db/index.js';
 import { emit } from './bus.js';
 import { requestRestart } from '../provisioning/index.js';
+import { reconcileCampaignRecovery } from './campaign-recovery.js';
 
 interface PendingRow {
     deployment_id: string;
@@ -40,7 +41,9 @@ async function runSweep(): Promise<SweepResult> {
     const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
     for (const row of pending) {
         if (!row.files_changed) continue; // A failed file write is not a restart request.
-        const operation = requestRestart(row.server, { kind: 'campaign', id: row.deployment_id });
+        const requested = requestRestart(row.server, { kind: 'campaign', id: row.deployment_id });
+        const operation =
+            reconcileCampaignRecovery(row.deployment_id, requested, row.files_changed) ?? requested;
         result.checked++;
         const state =
             operation.state === 'done'

src/routes/+page.svelte

@@ -391,6 +391,9 @@
                         (item) => item.server === server.name
                     )}
                     {@const operationIssue = operation && status.state === 'online'}
+                    {@const evidenceAt = operationIssue
+                        ? (operation.evidenceAt ?? operation.startedAt)
+                        : status.since}
                     {@const actionLabel = operationIssue ? 'Review operation' : status.actionLabel}
                     <div class="attention-row">
                         <div class="attention-identity">
@@ -412,9 +415,9 @@
                                 >{operationIssue ? operation.label : status.reason}</span
                             ><span class="concise-reason"
                                 >{operationIssue ? operation.label : status.shortReason}</span
-                            >{#if status.since}<span class="evidence-time" title={status.since}
+                            >{#if evidenceAt}<span class="evidence-time" title={evidenceAt}
                                     ><span class="evidence-separator">{' · '}</span>Evidence {elapsedSince(
-                                        status.since,
+                                        evidenceAt,
                                         $uiNow
                                     )} ago</span
                                 >{/if}

src/routes/api/campaigns/[id]/+server.ts

@@ -10,6 +10,13 @@ export async function GET({ params }: { params: { id: string } }) {
     const servers = db()
         .prepare('SELECT * FROM deployment_servers WHERE deployment_id = ? ORDER BY server')
         .all(params.id);
+    const operations = db()
+        .prepare(
+            `SELECT d.id,d.server,d.action,d.state,d.finished_at FROM provisioning_sources s
+            JOIN server_deployments d ON d.id=s.operation_id
+            WHERE s.kind='campaign' AND s.source_id=? ORDER BY d.started_at,d.rowid`
+        )
+        .all(params.id);
 
     // Don't ship blobs in the summary — the client asks for a single file when
     // it opens the diff viewer.
@@ -56,6 +63,7 @@ export async function GET({ params }: { params: { id: string } }) {
         fileContentsExpired,
         deployment: dep,
         servers,
+        operations,
         files,
         scripts,
         events,

src/routes/api/campaigns/[id]/resume/+server.ts

@@ -9,18 +9,42 @@ export function POST({ params }: { params: { id: string } }) {
             .prepare(
                 `SELECT ds.server, d.state FROM deployment_servers ds
             LEFT JOIN server_deployments d ON d.id = ds.provisioning_id
-            WHERE ds.deployment_id = ? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('held','manual')`
+            WHERE ds.deployment_id = ? AND ds.server_deleted_at IS NULL
+            AND (ds.restart_state IN ('held','manual') OR (ds.restart_state='failed' AND ds.files_changed>0))`
             )
             .all(params.id) as { server: string; state: string | null }[];
         for (const row of rows) {
             if (row.state && ['failed', 'interrupted', 'cancelled'].includes(row.state)) {
                 releaseRestartRequest(row.server, { kind: 'campaign', id: params.id });
+                // A failed health check may follow successful file application. A newer
+                // generation can have different files, so retry the retained desired bytes.
+                // Keep the original rollback image and refresh an existing compare guard.
+                db()
+                    .prepare(
+                        `UPDATE staged_file_changes SET state='pending',applied_at=NULL,
+                        preserve_before_image=1,expected_before_sha=CASE WHEN check_before=1 THEN content_hash ELSE expected_before_sha END
+                        WHERE source_kind='campaign' AND source_id=? AND server=? AND state='applied'`
+                    )
+                    .run(params.id, row.server);
+                db()
+                    .prepare(
+                        `UPDATE deployment_servers SET restarted_at=NULL,checked_at=NULL,players=NULL
+                        WHERE deployment_id=? AND server=?`
+                    )
+                    .run(params.id, row.server);
+                db()
+                    .prepare('INSERT INTO campaign_events(deployment_id,message) VALUES (?,?)')
+                    .run(
+                        params.id,
+                        `${row.server}: retry requested; previous operation retained in history.`
+                    );
             }
         }
         const result = db()
             .prepare(
                 `UPDATE deployment_servers SET restart_state = 'pending', error = NULL
-            WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('held','manual')`
+            WHERE deployment_id = ? AND server_deleted_at IS NULL
+            AND (restart_state IN ('held','manual') OR (restart_state='failed' AND files_changed>0))`
             )
             .run(params.id);
         if (result.changes)
@@ -28,7 +52,10 @@ export function POST({ params }: { params: { id: string } }) {
         return result.changes;
     })();
     if (!changed)
-        return json({ error: 'This campaign has no held changes to resume.' }, { status: 409 });
+        return json(
+            { error: 'This campaign has no held or failed activations to retry.' },
+            { status: 409 }
+        );
     void kick().catch((err) => console.error('[campaign] activation kick failed:', err));
     return json({ ok: true }, { status: 202 });
 }

src/routes/campaigns/[id]/+page.svelte

@@ -63,6 +63,13 @@
             rolled_back_by: string | null;
         };
         servers: ServerRow[];
+        operations: {
+            id: string;
+            server: string;
+            action: string;
+            state: string;
+            finished_at: string | null;
+        }[];
         files: FileRow[];
         events: ActivityEvent[];
         watchdogIncidentId?: string | null;
@@ -386,9 +393,16 @@
                             >All servers activated</span
                         >{/if}
                 </div>
-                {#if writable && detail.servers.some((server) => !server.server_deleted_at && ['held', 'manual'].includes(server.restart_state))}
+                {#if writable && detail.servers.some((server) => !server.server_deleted_at && (['held', 'manual'].includes(server.restart_state) || (server.restart_state === 'failed' && server.files_changed > 0)))}
                     <Button size="sm" onclick={resume} loading={resuming} disabled={resuming}
-                        >Resume when empty</Button
+                        >{detail.servers.some(
+                            (server) =>
+                                !server.server_deleted_at &&
+                                server.restart_state === 'failed' &&
+                                server.files_changed > 0
+                        )
+                            ? 'Retry when empty'
+                            : 'Resume when empty'}</Button
                     >
                 {/if}
             </div>
@@ -525,6 +539,20 @@
                                                         $page.url
                                                     )}>View provisioning</a
                                                 >{/if}
+                                            {#each detail.operations.filter((operation) => operation.server === s.server && operation.id !== s.provisioning_id) as operation (operation.id)}
+                                                <a
+                                                    href={contextHref(
+                                                        `/provisioning/${encodeURIComponent(operation.id)}`,
+                                                        $page.url
+                                                    )}
+                                                >
+                                                    Previous {operation.action}: {operation.state}
+                                                    {#if operation.finished_at}
+                                                        · <RelativeTime
+                                                            value={operation.finished_at}
+                                                        />{/if}
+                                                </a>
+                                            {/each}
                                         </div>
                                     </div>
                                 </td>

tests/unit/campaign-recovery.test.ts

@@ -0,0 +1,318 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { prepareNativeHost } from '../helpers/native-host.js';
+
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
+
+const server = 't5_zombie_theater_1';
+const campaign = 'recover-campaign';
+const started = '2026-10-19T18:38:00.000Z';
+const failedAt = '2026-10-19T18:48:42.000Z';
+const recoveredAt = '2026-10-19T21:31:00.000Z';
+let fixture: Fixture;
+
+beforeEach(async () => {
+    vi.resetModules();
+    vi.useFakeTimers({ toFake: ['Date'] });
+    vi.setSystemTime(started);
+    fixture = await newFixture([server]);
+    const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
+    await seedServerRegistry();
+    await prepareNativeHost([server]);
+});
+afterEach(async () => {
+    await fixture.cleanup();
+    vi.useRealTimers();
+});
+
+async function failedCampaign() {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
+    const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
+    const { finishDeployment } = await import('../../src/lib/server/provisioning/history.js');
+    const handle = db();
+    handle.prepare('UPDATE native_instances SET generation=5').run();
+    handle
+        .prepare("INSERT INTO deployments(id,action,started_at) VALUES (?,'promote',?)")
+        .run(campaign, started);
+    handle
+        .prepare(
+            "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES (?,?,2,'failed')"
+        )
+        .run(campaign, server);
+    for (const path of ['common.gsc', 'leaderboard.gsc']) {
+        queueFileChange({
+            sourceKind: 'campaign',
+            sourceId: campaign,
+            server,
+            kind: 'script',
+            relPath: path,
+            content: Buffer.from(`intended ${path}`),
+            inventorySource: campaign
+        });
+    }
+    const failed = requestRestart(server, { kind: 'campaign', id: campaign });
+    vi.setSystemTime(failedAt);
+    finishDeployment(failed.id, { ok: false, message: 'Artifact download timed out' });
+    return { handle, failed };
+}
+
+async function recoveredCampaign() {
+    const { handle, failed } = await failedCampaign();
+    const { beginDeployment, finishDeployment } =
+        await import('../../src/lib/server/provisioning/history.js');
+    const { captureRecipePlan, recipeStorage } =
+        await import('../../src/lib/server/fleet/launch-recipes.js');
+    const { recordScript } = await import('../../src/lib/server/fleet/inventory.js');
+    const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+    vi.setSystemTime('2026-10-19T21:29:00.000Z');
+    const recovery = beginDeployment(server, 'rebuild');
+    const instance = nativeInstance(server)!;
+    const files = handle
+        .prepare('SELECT kind,rel_path,content_hash FROM staged_file_changes WHERE source_id=?')
+        .all(campaign) as { kind: 'script'; rel_path: string; content_hash: string }[];
+    const recipe = captureRecipePlan('t5', 'theater');
+    for (const file of files) {
+        recipe.files.push({
+            kind: file.kind,
+            path: file.rel_path,
+            sha256: file.content_hash,
+            name: file.rel_path,
+            size: 10,
+            versionId: null,
+            version: null,
+            releaseVersion: null
+        });
+        recordScript(server, file.rel_path, null, file.content_hash, 10, 'native-create');
+    }
+    const stored = recipeStorage(recipe);
+    handle
+        .prepare(
+            'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES (?,?,?)'
+        )
+        .run(recovery, stored.recipe, stored.recipeSummary);
+    handle
+        .prepare(
+            `INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,payload,state,created_at,updated_at)
+        VALUES ('replacement',?,'instance.rebuild',?,5,'{"nextGeneration":6}','completed',?,?)`
+        )
+        .run(instance.host_id, instance.id, new Date().toISOString(), recoveredAt);
+    handle
+        .prepare(
+            'UPDATE native_provisioning SET operation_id=?,applied_at=?,verified_at=? WHERE deployment_id=?'
+        )
+        .run('replacement', recoveredAt, recoveredAt, recovery);
+    handle.prepare('UPDATE native_instances SET generation=6 WHERE id=?').run(instance.id);
+    handle
+        .prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?')
+        .run(new Date().toISOString(), recovery);
+    vi.setSystemTime(recoveredAt);
+    finishDeployment(recovery, { ok: true });
+    return { handle, failed, recovery, recipe, instance };
+}
+
+async function sweep() {
+    return (await import('../../src/lib/server/pipeline/sweep.js')).sweep();
+}
+
+describe('campaign recovery', () => {
+    it('adopts a verified matching rebuild, completes the campaign, and retains both operations and failure history', async () => {
+        const { handle, failed, recovery } = await recoveredCampaign();
+        const original = handle
+            .prepare('SELECT * FROM server_deployments WHERE id=?')
+            .get(failed.id);
+        const events = handle
+            .prepare('SELECT * FROM server_deployment_events WHERE deployment_id=?')
+            .all(failed.id);
+        const { fleetOperations, serverOperations } =
+            await import('../../src/lib/server/fleet/overview.js');
+        expect(fleetOperations().find((op) => op.kind === 'campaign')).toMatchObject({
+            state: 'attention',
+            evidenceAt: failedAt
+        });
+        expect(await sweep()).toMatchObject({ healthy: 1, failed: 0 });
+        expect(
+            handle
+                .prepare(
+                    'SELECT restart_state,provisioning_id,error,checked_at,restarted_at FROM deployment_servers'
+                )
+                .get()
+        ).toEqual({
+            restart_state: 'done',
+            provisioning_id: recovery,
+            error: null,
+            checked_at: recoveredAt,
+            restarted_at: '2026-10-19T21:29:00.000Z'
+        });
+        expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({
+            finished_at: recoveredAt
+        });
+        expect(fleetOperations()).toEqual([]);
+        expect(serverOperations(server).find((op) => op.kind === 'campaign')).toMatchObject({
+            state: 'done',
+            evidenceAt: recoveredAt
+        });
+        expect(
+            handle.prepare('SELECT * FROM server_deployments WHERE id=?').get(failed.id)
+        ).toEqual(original);
+        expect(
+            handle
+                .prepare('SELECT * FROM server_deployment_events WHERE deployment_id=?')
+                .all(failed.id)
+        ).toEqual(events);
+        expect(
+            handle
+                .prepare(
+                    'SELECT operation_id FROM provisioning_sources WHERE source_id=? ORDER BY rowid'
+                )
+                .all(campaign)
+        ).toEqual([{ operation_id: failed.id }, { operation_id: recovery }]);
+        expect(handle.prepare('SELECT state FROM staged_file_changes').all()).toEqual([
+            { state: 'applied' },
+            { state: 'applied' }
+        ]);
+        const { GET } = await import('../../src/routes/api/campaigns/[id]/+server.js');
+        expect((await (await GET({ params: { id: campaign } })).json()).operations).toEqual([
+            expect.objectContaining({ id: failed.id, state: 'failed', finished_at: failedAt }),
+            expect.objectContaining({ id: recovery, state: 'done', finished_at: recoveredAt })
+        ]);
+        await sweep();
+        expect(
+            handle
+                .prepare("SELECT * FROM campaign_events WHERE message LIKE '%recovered by%'")
+                .all()
+        ).toHaveLength(1);
+    });
+
+    it.each([
+        'recipe mismatch',
+        'inventory mismatch',
+        'missing recipe',
+        'missing file',
+        'stale generation',
+        'wrong instance',
+        'wrong host',
+        'unverified',
+        'failed receipt',
+        'unrelated restart',
+        'older rebuild',
+        'cancelled files',
+        'deleted target',
+        'staged override mismatch'
+    ])('keeps attention for %s', async (condition) => {
+        const { handle, failed, recovery, recipe } = await recoveredCampaign();
+        if (condition === 'recipe mismatch') {
+            recipe.files.find((f) => f.path === 'leaderboard.gsc')!.sha256 = 'other';
+            handle
+                .prepare('UPDATE server_launch_recipes SET recipe_summary=?')
+                .run(JSON.stringify(recipe));
+        }
+        if (condition === 'inventory mismatch')
+            handle
+                .prepare(
+                    "UPDATE server_scripts SET sha256='other' WHERE rel_path='leaderboard.gsc'"
+                )
+                .run();
+        if (condition === 'missing recipe')
+            handle.prepare('DELETE FROM server_launch_recipes').run();
+        if (condition === 'missing file')
+            handle
+                .prepare("DELETE FROM staged_file_changes WHERE rel_path='leaderboard.gsc'")
+                .run();
+        if (condition === 'stale generation')
+            handle.prepare('UPDATE native_instances SET generation=7').run();
+        if (condition === 'wrong instance')
+            handle.prepare("UPDATE native_instances SET server='replacement-name'").run();
+        if (condition === 'wrong host')
+            handle.prepare("UPDATE native_instances SET host_id='other-host'").run();
+        if (condition === 'unverified')
+            handle
+                .prepare('UPDATE native_provisioning SET verified_at=NULL WHERE deployment_id=?')
+                .run(recovery);
+        if (condition === 'failed receipt')
+            handle.prepare("UPDATE ec_host_operations SET state='failed'").run();
+        if (condition === 'unrelated restart') {
+            handle
+                .prepare("UPDATE server_deployments SET action='restart' WHERE id=?")
+                .run(recovery);
+            handle.prepare("UPDATE ec_host_operations SET kind='instance.restart'").run();
+            handle.prepare('UPDATE native_instances SET generation=5').run();
+        }
+        if (condition === 'older rebuild')
+            handle
+                .prepare('UPDATE server_deployments SET started_at=? WHERE id=?')
+                .run(started, recovery);
+        if (condition === 'cancelled files')
+            handle.prepare("UPDATE staged_file_changes SET state='cancelled'").run();
+        if (condition === 'deleted target')
+            handle.prepare('UPDATE deployment_servers SET server_deleted_at=?').run(recoveredAt);
+        if (condition === 'staged override mismatch')
+            handle.prepare('UPDATE native_provisioning SET files_json=? WHERE deployment_id=?').run(
+                JSON.stringify([
+                    {
+                        path: 'runtime/plutonium/storage/t5/raw/scripts/sp/leaderboard.gsc',
+                        sha256: 'other'
+                    }
+                ]),
+                recovery
+            );
+        await sweep();
+        expect(
+            handle.prepare('SELECT restart_state,provisioning_id FROM deployment_servers').get()
+        ).toEqual({ restart_state: 'failed', provisioning_id: failed.id });
+        expect(
+            handle
+                .prepare("SELECT * FROM campaign_events WHERE message LIKE '%recovered by%'")
+                .all()
+        ).toEqual([]);
+        if (condition !== 'deleted target') {
+            expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({
+                finished_at: null
+            });
+            const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
+            expect(fleetOperations().find((op) => op.kind === 'campaign')).toMatchObject({
+                state: 'attention',
+                evidenceAt: failedAt
+            });
+        }
+    });
+
+    it('does not accept fresh healthy status without a successful replacement operation', async () => {
+        const { handle, failed } = await failedCampaign();
+        handle
+            .prepare(
+                "UPDATE servers SET last_seen_at=?,last_checked_at=?,last_error=NULL,container_state='running'"
+            )
+            .run(recoveredAt, recoveredAt);
+        await sweep();
+        expect(
+            handle.prepare('SELECT restart_state,provisioning_id FROM deployment_servers').get()
+        ).toEqual({ restart_state: 'failed', provisioning_id: failed.id });
+        const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
+        expect(
+            fleetOperations()
+                .filter((op) => op.state === 'attention')
+                .map((op) => op.evidenceAt)
+        ).toEqual([failedAt, failedAt]);
+    });
+
+    it('waits for every remaining target before completing the campaign', async () => {
+        const { handle } = await recoveredCampaign();
+        handle
+            .prepare(
+                "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES (?,'other',1,'held')"
+            )
+            .run(campaign);
+        await sweep();
+        expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({
+            finished_at: null
+        });
+        handle
+            .prepare("UPDATE deployment_servers SET restart_state='done' WHERE server='other'")
+            .run();
+        await sweep();
+        expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({
+            finished_at: recoveredAt
+        });
+    });
+});

tests/unit/campaign-resume.test.ts

@@ -57,7 +57,7 @@ async function resume(id = campaign) {
 }
 
 describe('campaign resume API', () => {
-    it.each(['held', 'manual'])(
+    it.each(['held', 'manual', 'failed'])(
         'queues %s rollback changes without touching live files',
         async (state) => {
             await heldRollback(state);
@@ -93,7 +93,7 @@ describe('campaign resume API', () => {
     it.each(['cancelled', 'failed', 'interrupted'])(
         'releases an old %s operation so the worker creates fresh restart intent',
         async (state) => {
-            await heldRollback();
+            await heldRollback('failed');
             const { requestRestart } =
                 await import('../../src/lib/server/provisioning/requests.js');
             const { db } = await import('../../src/lib/server/db/index.js');
@@ -101,6 +101,11 @@ describe('campaign resume API', () => {
             db()
                 .prepare('UPDATE server_deployments SET state = ?, finished_at = ? WHERE id = ?')
                 .run(state, new Date().toISOString(), old.id);
+            db()
+                .prepare(
+                    'UPDATE deployment_servers SET restarted_at=?,checked_at=?,players=3 WHERE deployment_id=?'
+                )
+                .run(new Date().toISOString(), new Date().toISOString(), campaign);
             expect((await resume()).status).toBe(202);
             expect(
                 db()
@@ -113,7 +118,14 @@ describe('campaign resume API', () => {
             await sweep();
             const next = requestRestart(server, { kind: 'campaign', id: campaign });
             expect(next.id).not.toBe(old.id);
-            expect(next).toMatchObject({ state: 'waiting', submitted_at: null });
+            expect(next).toMatchObject({ state: 'waiting', action: 'rebuild', submitted_at: null });
+            expect(
+                db()
+                    .prepare(
+                        'SELECT restarted_at,checked_at,players FROM deployment_servers WHERE deployment_id=?'
+                    )
+                    .get(campaign)
+            ).toEqual({ restarted_at: null, checked_at: null, players: null });
             expect(
                 db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.id)
             ).toEqual({ state });
@@ -174,7 +186,7 @@ describe('campaign resume API', () => {
             const response = await resume();
             expect(response.status).toBe(409);
             expect(await response.json()).toEqual({
-                error: 'This campaign has no held changes to resume.'
+                error: 'This campaign has no held or failed activations to retry.'
             });
             expect(kick).not.toHaveBeenCalled();
             expect(db().prepare('SELECT * FROM server_deployments').all()).toEqual([]);
@@ -193,4 +205,11 @@ describe('campaign resume API', () => {
             }
         }
     );
+
+    it('does not retry a failed file-staging attempt with no activation work', async () => {
+        await heldRollback('failed');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db().prepare('UPDATE deployment_servers SET files_changed=0').run();
+        expect((await resume()).status).toBe(409);
+    });
 });

tests/unit/native-provisioning.test.ts

@@ -1948,6 +1948,145 @@ describe('native instance provisioning', () => {
         expect(nativeInstance(job.name)?.generation).toBe(3);
     });
 
+    it.each(['pending', 'applied'])(
+        'retries %s files from a failed campaign on the current generation through occupancy and startup admission',
+        async (fileState) => {
+            const job = await queue();
+            await advance();
+            await acknowledge(job.deploymentId);
+            const { db } = await import('../../src/lib/server/db/index.js');
+            const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
+            const { requestRestart } =
+                await import('../../src/lib/server/provisioning/requests.js');
+            const { finishDeployment } =
+                await import('../../src/lib/server/provisioning/history.js');
+            const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+            const { processProvisioning } =
+                await import('../../src/lib/server/provisioning/worker.js');
+            db()
+                .prepare(
+                    "INSERT INTO deployments(id,action,started_at) VALUES('retry','promote',?)"
+                )
+                .run(new Date().toISOString());
+            db()
+                .prepare(
+                    "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('retry',?,1,'failed')"
+                )
+                .run(job.name);
+            queueFileChange({
+                sourceKind: 'campaign',
+                sourceId: 'retry',
+                server: job.name,
+                kind: 'script',
+                relPath: 'retry.gsc',
+                content: Buffer.from('desired campaign bytes'),
+                inventorySource: 'test'
+            });
+            const failed = requestRestart(job.name, { kind: 'campaign', id: 'retry' });
+            finishDeployment(failed.id, { ok: false, message: 'Download timeout' });
+            db()
+                .prepare(
+                    "UPDATE staged_file_changes SET state=? WHERE source_kind='campaign' AND source_id='retry'"
+                )
+                .run(fileState);
+            const { queueNativeRebuild } =
+                await import('../../src/lib/server/provisioning/native.js');
+            const unrelated = queueNativeRebuild(job.name, 'ui');
+            if (!unrelated.ok) throw new Error(unrelated.message);
+            await advance();
+            await acknowledge(unrelated.deploymentId);
+            await sweep();
+            expect(
+                db()
+                    .prepare(
+                        "SELECT restart_state FROM deployment_servers WHERE deployment_id='retry'"
+                    )
+                    .get()
+            ).toEqual({ restart_state: 'failed' });
+
+            const { fleetSettings, saveFleetSettings } =
+                await import('../../src/lib/server/settings.js');
+            saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
+            const { queueCreateServer } =
+                await import('../../src/lib/server/provisioning/instances.js');
+            const other = queueCreateServer({
+                game: 't5',
+                gameMode: 'zombie',
+                mapSlug: 'theater',
+                hostId,
+                libraryId,
+                image,
+                plutoKey: 'second-key'
+            });
+            if (!other.ok) throw new Error(other.message);
+            await advance();
+            expect(await operationFor(other.deploymentId)).not.toBeNull();
+
+            db()
+                .prepare("UPDATE deployment_servers SET restarted_at=? WHERE deployment_id='retry'")
+                .run(new Date().toISOString());
+            const { POST } = await import('../../src/routes/api/campaigns/[id]/resume/+server.js');
+            expect(POST({ params: { id: 'retry' } }).status).toBe(202);
+            await sweep();
+            const next = requestRestart(job.name, { kind: 'campaign', id: 'retry' });
+            expect(next).toMatchObject({ action: 'rebuild', state: 'waiting', submitted_at: null });
+            expect(next.id).not.toBe(failed.id);
+            expect(
+                db()
+                    .prepare('SELECT generation FROM native_provisioning WHERE deployment_id=?')
+                    .get(next.id)
+            ).toEqual({ generation: 2 });
+            await processProvisioning();
+            const staged = db()
+                .prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?')
+                .get(next.id) as { stage_operation_id: string };
+            const { reportHostOperation, getHostOperation } =
+                await import('../../src/lib/server/hosts/index.js');
+            reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' });
+            await processProvisioning();
+            expect(await operationFor(next.id)).toBeNull();
+            expect((await detail(next.id)).stage).toContain('startup capacity');
+            remote.players = 2;
+            await acknowledge(other.deploymentId);
+            await processProvisioning();
+            expect(await operationFor(next.id)).toBeNull();
+            expect((await detail(next.id)).stage).toContain('player');
+            remote.players = 0;
+            await processProvisioning();
+            const activation = (await operationFor(next.id))!;
+            expect(activation, (await detail(next.id)).stage).toMatchObject({
+                kind: 'instance.rebuild',
+                generation: 2,
+                payload: { nextGeneration: 3, stageId: next.id }
+            });
+            const files = getHostOperation(staged.stage_operation_id)!.payload.files as {
+                id: number;
+                path: string;
+            }[];
+            reportHostOperation(hostId, activation.id, {
+                state: 'completed',
+                result: { before: files.map((file) => ({ ...file, contentBase64: null })) }
+            });
+            await processProvisioning();
+            await sweep();
+            expect(
+                db()
+                    .prepare(
+                        "SELECT restart_state,provisioning_id FROM deployment_servers WHERE deployment_id='retry'"
+                    )
+                    .get()
+            ).toEqual({ restart_state: 'done', provisioning_id: next.id });
+            expect((await detail(failed.id)).state).toBe('failed');
+            expect(
+                db()
+                    .prepare(
+                        "SELECT operation_id FROM provisioning_sources WHERE kind='campaign' AND source_id='retry' ORDER BY rowid"
+                    )
+                    .all()
+            ).toEqual([{ operation_id: failed.id }, { operation_id: next.id }]);
+        }
+    );
+
     it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => {
         const job = await queue();
         await advance();

tests/unit/provisioning-history-performance.test.ts

@@ -87,7 +87,7 @@ describe('provisioning reads with retained lifecycle history', () => {
         fleetOperations(now);
         const queries = prepare.mock.calls
             .map(([sql]) => sql)
-            .filter((sql) => sql.includes('server_deployments'));
+            .filter((sql) => sql.includes('server_deployments') && sql.includes('UNION'));
         prepare.mockRestore();
         expect(queries).toHaveLength(1);
         const statement = handle.prepare(queries[0]);
@@ -98,7 +98,8 @@ describe('provisioning reads with retained lifecycle history', () => {
             'state',
             'stage',
             'error',
-            'started_at'
+            'started_at',
+            'evidence_at'
         ]);
         const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all() as {
             detail: string;

tests/unit/ui-fleet-attention.test.ts

@@ -0,0 +1,90 @@
+import { expect, it, vi } from 'vitest';
+import { render } from 'svelte/server';
+import { writable } from 'svelte/store';
+import type { ComponentProps } from 'svelte';
+import Page from '../../src/routes/+page.svelte';
+import { syncServerTime } from '../../src/lib/ui/stores/servers.js';
+import type { FleetServer } from '../../src/lib/fleet-overview.js';
+
+vi.mock('$app/navigation', () => ({
+    afterNavigate: vi.fn(),
+    replaceState: vi.fn(),
+    goto: vi.fn(),
+    beforeNavigate: vi.fn()
+}));
+vi.mock('$app/stores', () => ({
+    page: writable({ url: new URL('http://localhost/'), data: { user: { role: 'viewer' } } })
+}));
+
+const now = Date.parse('2026-10-19T22:46:00.000Z');
+const healthAt = new Date(now - 8000).toISOString();
+const failedAt = '2026-10-19T18:48:42.000Z';
+const server: FleetServer = {
+    name: 't5_zombie_sumpf_1',
+    game: 't5',
+    map: 'zombie_sumpf',
+    last_map: 'zombie_sumpf',
+    last_players: 0,
+    last_gametype: 'zom',
+    last_seen_at: healthAt,
+    last_checked_at: healthAt,
+    container_state: 'running',
+    container_checked_at: healthAt,
+    last_error: null,
+    roster: [],
+    health: 'online',
+    held: [],
+    active_since: null,
+    inactive_since: null,
+    game_started_at: null,
+    game_ended_at: null
+};
+
+it.each([true, false])(
+    'dates the displayed attention reason when the server is healthy=%s',
+    (healthy) => {
+        syncServerTime(now);
+        const data = {
+            user: null,
+            setupRequired: false,
+            fleetSnapshot: {
+                serverNow: now,
+                hostAttention: 0,
+                reservedNames: [],
+                servers: [
+                    healthy ? server : { ...server, health: 'offline', container_state: 'stopped' }
+                ],
+                integrations: {
+                    native: true,
+                    hostsReady: 1,
+                    hostsTotal: 1,
+                    rcon: true,
+                    rconTimeoutMs: 2000,
+                    defaultRconHost: 'localhost'
+                },
+                operations: [
+                    {
+                        key: 'campaign:one',
+                        group: 'campaign:one',
+                        kind: 'campaign',
+                        server: server.name,
+                        label: 'Update needs attention',
+                        detail: 'Download timed out',
+                        href: '/campaigns/one',
+                        state: 'attention',
+                        startedAt: '2026-10-19T18:38:00.000Z',
+                        evidenceAt: failedAt
+                    }
+                ]
+            }
+            // Rendering needs the public Fleet fields, without the registry's storage metadata.
+        } as unknown as ComponentProps<typeof Page>['data'];
+        const body = render(Page, { props: { data } }).body;
+        const evidence = body.match(/class="evidence-time[^"]*" title="([^"]+)"/)?.[1];
+        expect(evidence).toBe(healthy ? failedAt : healthAt);
+        if (healthy) {
+            expect(body).toContain('Update needs attention');
+            expect(body).toContain('Evidence 3h 57m ago');
+        }
+    }
+);