Reconcile recovered campaign targets (Refs #166)

tests/unit/campaign-recovery.test.ts

@@ -0,0 +1,318 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { prepareNativeHost } from '../helpers/native-host.js';
+
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
+
+const server = 't5_zombie_theater_1';
+const campaign = 'recover-campaign';
+const started = '2026-10-19T18:38:00.000Z';
+const failedAt = '2026-10-19T18:48:42.000Z';
+const recoveredAt = '2026-10-19T21:31:00.000Z';
+let fixture: Fixture;
+
+beforeEach(async () => {
+    vi.resetModules();
+    vi.useFakeTimers({ toFake: ['Date'] });
+    vi.setSystemTime(started);
+    fixture = await newFixture([server]);
+    const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
+    await seedServerRegistry();
+    await prepareNativeHost([server]);
+});
+afterEach(async () => {
+    await fixture.cleanup();
+    vi.useRealTimers();
+});
+
+async function failedCampaign() {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
+    const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
+    const { finishDeployment } = await import('../../src/lib/server/provisioning/history.js');
+    const handle = db();
+    handle.prepare('UPDATE native_instances SET generation=5').run();
+    handle
+        .prepare("INSERT INTO deployments(id,action,started_at) VALUES (?,'promote',?)")
+        .run(campaign, started);
+    handle
+        .prepare(
+            "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES (?,?,2,'failed')"
+        )
+        .run(campaign, server);
+    for (const path of ['common.gsc', 'leaderboard.gsc']) {
+        queueFileChange({
+            sourceKind: 'campaign',
+            sourceId: campaign,
+            server,
+            kind: 'script',
+            relPath: path,
+            content: Buffer.from(`intended ${path}`),
+            inventorySource: campaign
+        });
+    }
+    const failed = requestRestart(server, { kind: 'campaign', id: campaign });
+    vi.setSystemTime(failedAt);
+    finishDeployment(failed.id, { ok: false, message: 'Artifact download timed out' });
+    return { handle, failed };
+}
+
+async function recoveredCampaign() {
+    const { handle, failed } = await failedCampaign();
+    const { beginDeployment, finishDeployment } =
+        await import('../../src/lib/server/provisioning/history.js');
+    const { captureRecipePlan, recipeStorage } =
+        await import('../../src/lib/server/fleet/launch-recipes.js');
+    const { recordScript } = await import('../../src/lib/server/fleet/inventory.js');
+    const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+    vi.setSystemTime('2026-10-19T21:29:00.000Z');
+    const recovery = beginDeployment(server, 'rebuild');
+    const instance = nativeInstance(server)!;
+    const files = handle
+        .prepare('SELECT kind,rel_path,content_hash FROM staged_file_changes WHERE source_id=?')
+        .all(campaign) as { kind: 'script'; rel_path: string; content_hash: string }[];
+    const recipe = captureRecipePlan('t5', 'theater');
+    for (const file of files) {
+        recipe.files.push({
+            kind: file.kind,
+            path: file.rel_path,
+            sha256: file.content_hash,
+            name: file.rel_path,
+            size: 10,
+            versionId: null,
+            version: null,
+            releaseVersion: null
+        });
+        recordScript(server, file.rel_path, null, file.content_hash, 10, 'native-create');
+    }
+    const stored = recipeStorage(recipe);
+    handle
+        .prepare(
+            'INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES (?,?,?)'
+        )
+        .run(recovery, stored.recipe, stored.recipeSummary);
+    handle
+        .prepare(
+            `INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,payload,state,created_at,updated_at)
+        VALUES ('replacement',?,'instance.rebuild',?,5,'{"nextGeneration":6}','completed',?,?)`
+        )
+        .run(instance.host_id, instance.id, new Date().toISOString(), recoveredAt);
+    handle
+        .prepare(
+            'UPDATE native_provisioning SET operation_id=?,applied_at=?,verified_at=? WHERE deployment_id=?'
+        )
+        .run('replacement', recoveredAt, recoveredAt, recovery);
+    handle.prepare('UPDATE native_instances SET generation=6 WHERE id=?').run(instance.id);
+    handle
+        .prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?')
+        .run(new Date().toISOString(), recovery);
+    vi.setSystemTime(recoveredAt);
+    finishDeployment(recovery, { ok: true });
+    return { handle, failed, recovery, recipe, instance };
+}
+
+async function sweep() {
+    return (await import('../../src/lib/server/pipeline/sweep.js')).sweep();
+}
+
+describe('campaign recovery', () => {
+    it('adopts a verified matching rebuild, completes the campaign, and retains both operations and failure history', async () => {
+        const { handle, failed, recovery } = await recoveredCampaign();
+        const original = handle
+            .prepare('SELECT * FROM server_deployments WHERE id=?')
+            .get(failed.id);
+        const events = handle
+            .prepare('SELECT * FROM server_deployment_events WHERE deployment_id=?')
+            .all(failed.id);
+        const { fleetOperations, serverOperations } =
+            await import('../../src/lib/server/fleet/overview.js');
+        expect(fleetOperations().find((op) => op.kind === 'campaign')).toMatchObject({
+            state: 'attention',
+            evidenceAt: failedAt
+        });
+        expect(await sweep()).toMatchObject({ healthy: 1, failed: 0 });
+        expect(
+            handle
+                .prepare(
+                    'SELECT restart_state,provisioning_id,error,checked_at,restarted_at FROM deployment_servers'
+                )
+                .get()
+        ).toEqual({
+            restart_state: 'done',
+            provisioning_id: recovery,
+            error: null,
+            checked_at: recoveredAt,
+            restarted_at: '2026-10-19T21:29:00.000Z'
+        });
+        expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({
+            finished_at: recoveredAt
+        });
+        expect(fleetOperations()).toEqual([]);
+        expect(serverOperations(server).find((op) => op.kind === 'campaign')).toMatchObject({
+            state: 'done',
+            evidenceAt: recoveredAt
+        });
+        expect(
+            handle.prepare('SELECT * FROM server_deployments WHERE id=?').get(failed.id)
+        ).toEqual(original);
+        expect(
+            handle
+                .prepare('SELECT * FROM server_deployment_events WHERE deployment_id=?')
+                .all(failed.id)
+        ).toEqual(events);
+        expect(
+            handle
+                .prepare(
+                    'SELECT operation_id FROM provisioning_sources WHERE source_id=? ORDER BY rowid'
+                )
+                .all(campaign)
+        ).toEqual([{ operation_id: failed.id }, { operation_id: recovery }]);
+        expect(handle.prepare('SELECT state FROM staged_file_changes').all()).toEqual([
+            { state: 'applied' },
+            { state: 'applied' }
+        ]);
+        const { GET } = await import('../../src/routes/api/campaigns/[id]/+server.js');
+        expect((await (await GET({ params: { id: campaign } })).json()).operations).toEqual([
+            expect.objectContaining({ id: failed.id, state: 'failed', finished_at: failedAt }),
+            expect.objectContaining({ id: recovery, state: 'done', finished_at: recoveredAt })
+        ]);
+        await sweep();
+        expect(
+            handle
+                .prepare("SELECT * FROM campaign_events WHERE message LIKE '%recovered by%'")
+                .all()
+        ).toHaveLength(1);
+    });
+
+    it.each([
+        'recipe mismatch',
+        'inventory mismatch',
+        'missing recipe',
+        'missing file',
+        'stale generation',
+        'wrong instance',
+        'wrong host',
+        'unverified',
+        'failed receipt',
+        'unrelated restart',
+        'older rebuild',
+        'cancelled files',
+        'deleted target',
+        'staged override mismatch'
+    ])('keeps attention for %s', async (condition) => {
+        const { handle, failed, recovery, recipe } = await recoveredCampaign();
+        if (condition === 'recipe mismatch') {
+            recipe.files.find((f) => f.path === 'leaderboard.gsc')!.sha256 = 'other';
+            handle
+                .prepare('UPDATE server_launch_recipes SET recipe_summary=?')
+                .run(JSON.stringify(recipe));
+        }
+        if (condition === 'inventory mismatch')
+            handle
+                .prepare(
+                    "UPDATE server_scripts SET sha256='other' WHERE rel_path='leaderboard.gsc'"
+                )
+                .run();
+        if (condition === 'missing recipe')
+            handle.prepare('DELETE FROM server_launch_recipes').run();
+        if (condition === 'missing file')
+            handle
+                .prepare("DELETE FROM staged_file_changes WHERE rel_path='leaderboard.gsc'")
+                .run();
+        if (condition === 'stale generation')
+            handle.prepare('UPDATE native_instances SET generation=7').run();
+        if (condition === 'wrong instance')
+            handle.prepare("UPDATE native_instances SET server='replacement-name'").run();
+        if (condition === 'wrong host')
+            handle.prepare("UPDATE native_instances SET host_id='other-host'").run();
+        if (condition === 'unverified')
+            handle
+                .prepare('UPDATE native_provisioning SET verified_at=NULL WHERE deployment_id=?')
+                .run(recovery);
+        if (condition === 'failed receipt')
+            handle.prepare("UPDATE ec_host_operations SET state='failed'").run();
+        if (condition === 'unrelated restart') {
+            handle
+                .prepare("UPDATE server_deployments SET action='restart' WHERE id=?")
+                .run(recovery);
+            handle.prepare("UPDATE ec_host_operations SET kind='instance.restart'").run();
+            handle.prepare('UPDATE native_instances SET generation=5').run();
+        }
+        if (condition === 'older rebuild')
+            handle
+                .prepare('UPDATE server_deployments SET started_at=? WHERE id=?')
+                .run(started, recovery);
+        if (condition === 'cancelled files')
+            handle.prepare("UPDATE staged_file_changes SET state='cancelled'").run();
+        if (condition === 'deleted target')
+            handle.prepare('UPDATE deployment_servers SET server_deleted_at=?').run(recoveredAt);
+        if (condition === 'staged override mismatch')
+            handle.prepare('UPDATE native_provisioning SET files_json=? WHERE deployment_id=?').run(
+                JSON.stringify([
+                    {
+                        path: 'runtime/plutonium/storage/t5/raw/scripts/sp/leaderboard.gsc',
+                        sha256: 'other'
+                    }
+                ]),
+                recovery
+            );
+        await sweep();
+        expect(
+            handle.prepare('SELECT restart_state,provisioning_id FROM deployment_servers').get()
+        ).toEqual({ restart_state: 'failed', provisioning_id: failed.id });
+        expect(
+            handle
+                .prepare("SELECT * FROM campaign_events WHERE message LIKE '%recovered by%'")
+                .all()
+        ).toEqual([]);
+        if (condition !== 'deleted target') {
+            expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({
+                finished_at: null
+            });
+            const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
+            expect(fleetOperations().find((op) => op.kind === 'campaign')).toMatchObject({
+                state: 'attention',
+                evidenceAt: failedAt
+            });
+        }
+    });
+
+    it('does not accept fresh healthy status without a successful replacement operation', async () => {
+        const { handle, failed } = await failedCampaign();
+        handle
+            .prepare(
+                "UPDATE servers SET last_seen_at=?,last_checked_at=?,last_error=NULL,container_state='running'"
+            )
+            .run(recoveredAt, recoveredAt);
+        await sweep();
+        expect(
+            handle.prepare('SELECT restart_state,provisioning_id FROM deployment_servers').get()
+        ).toEqual({ restart_state: 'failed', provisioning_id: failed.id });
+        const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
+        expect(
+            fleetOperations()
+                .filter((op) => op.state === 'attention')
+                .map((op) => op.evidenceAt)
+        ).toEqual([failedAt, failedAt]);
+    });
+
+    it('waits for every remaining target before completing the campaign', async () => {
+        const { handle } = await recoveredCampaign();
+        handle
+            .prepare(
+                "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES (?,'other',1,'held')"
+            )
+            .run(campaign);
+        await sweep();
+        expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({
+            finished_at: null
+        });
+        handle
+            .prepare("UPDATE deployment_servers SET restart_state='done' WHERE server='other'")
+            .run();
+        await sweep();
+        expect(handle.prepare('SELECT finished_at FROM deployments').get()).toEqual({
+            finished_at: recoveredAt
+        });
+    });
+});

tests/unit/campaign-resume.test.ts

@@ -57,7 +57,7 @@ async function resume(id = campaign) {
 }
 
 describe('campaign resume API', () => {
-    it.each(['held', 'manual'])(
+    it.each(['held', 'manual', 'failed'])(
         'queues %s rollback changes without touching live files',
         async (state) => {
             await heldRollback(state);
@@ -93,7 +93,7 @@ describe('campaign resume API', () => {
     it.each(['cancelled', 'failed', 'interrupted'])(
         'releases an old %s operation so the worker creates fresh restart intent',
         async (state) => {
-            await heldRollback();
+            await heldRollback('failed');
             const { requestRestart } =
                 await import('../../src/lib/server/provisioning/requests.js');
             const { db } = await import('../../src/lib/server/db/index.js');
@@ -101,6 +101,11 @@ describe('campaign resume API', () => {
             db()
                 .prepare('UPDATE server_deployments SET state = ?, finished_at = ? WHERE id = ?')
                 .run(state, new Date().toISOString(), old.id);
+            db()
+                .prepare(
+                    'UPDATE deployment_servers SET restarted_at=?,checked_at=?,players=3 WHERE deployment_id=?'
+                )
+                .run(new Date().toISOString(), new Date().toISOString(), campaign);
             expect((await resume()).status).toBe(202);
             expect(
                 db()
@@ -113,7 +118,14 @@ describe('campaign resume API', () => {
             await sweep();
             const next = requestRestart(server, { kind: 'campaign', id: campaign });
             expect(next.id).not.toBe(old.id);
-            expect(next).toMatchObject({ state: 'waiting', submitted_at: null });
+            expect(next).toMatchObject({ state: 'waiting', action: 'rebuild', submitted_at: null });
+            expect(
+                db()
+                    .prepare(
+                        'SELECT restarted_at,checked_at,players FROM deployment_servers WHERE deployment_id=?'
+                    )
+                    .get(campaign)
+            ).toEqual({ restarted_at: null, checked_at: null, players: null });
             expect(
                 db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.id)
             ).toEqual({ state });
@@ -174,7 +186,7 @@ describe('campaign resume API', () => {
             const response = await resume();
             expect(response.status).toBe(409);
             expect(await response.json()).toEqual({
-                error: 'This campaign has no held changes to resume.'
+                error: 'This campaign has no held or failed activations to retry.'
             });
             expect(kick).not.toHaveBeenCalled();
             expect(db().prepare('SELECT * FROM server_deployments').all()).toEqual([]);
@@ -193,4 +205,11 @@ describe('campaign resume API', () => {
             }
         }
     );
+
+    it('does not retry a failed file-staging attempt with no activation work', async () => {
+        await heldRollback('failed');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db().prepare('UPDATE deployment_servers SET files_changed=0').run();
+        expect((await resume()).status).toBe(409);
+    });
 });

tests/unit/native-provisioning.test.ts

@@ -1948,6 +1948,145 @@ describe('native instance provisioning', () => {
         expect(nativeInstance(job.name)?.generation).toBe(3);
     });
 
+    it.each(['pending', 'applied'])(
+        'retries %s files from a failed campaign on the current generation through occupancy and startup admission',
+        async (fileState) => {
+            const job = await queue();
+            await advance();
+            await acknowledge(job.deploymentId);
+            const { db } = await import('../../src/lib/server/db/index.js');
+            const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
+            const { requestRestart } =
+                await import('../../src/lib/server/provisioning/requests.js');
+            const { finishDeployment } =
+                await import('../../src/lib/server/provisioning/history.js');
+            const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
+            const { processProvisioning } =
+                await import('../../src/lib/server/provisioning/worker.js');
+            db()
+                .prepare(
+                    "INSERT INTO deployments(id,action,started_at) VALUES('retry','promote',?)"
+                )
+                .run(new Date().toISOString());
+            db()
+                .prepare(
+                    "INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('retry',?,1,'failed')"
+                )
+                .run(job.name);
+            queueFileChange({
+                sourceKind: 'campaign',
+                sourceId: 'retry',
+                server: job.name,
+                kind: 'script',
+                relPath: 'retry.gsc',
+                content: Buffer.from('desired campaign bytes'),
+                inventorySource: 'test'
+            });
+            const failed = requestRestart(job.name, { kind: 'campaign', id: 'retry' });
+            finishDeployment(failed.id, { ok: false, message: 'Download timeout' });
+            db()
+                .prepare(
+                    "UPDATE staged_file_changes SET state=? WHERE source_kind='campaign' AND source_id='retry'"
+                )
+                .run(fileState);
+            const { queueNativeRebuild } =
+                await import('../../src/lib/server/provisioning/native.js');
+            const unrelated = queueNativeRebuild(job.name, 'ui');
+            if (!unrelated.ok) throw new Error(unrelated.message);
+            await advance();
+            await acknowledge(unrelated.deploymentId);
+            await sweep();
+            expect(
+                db()
+                    .prepare(
+                        "SELECT restart_state FROM deployment_servers WHERE deployment_id='retry'"
+                    )
+                    .get()
+            ).toEqual({ restart_state: 'failed' });
+
+            const { fleetSettings, saveFleetSettings } =
+                await import('../../src/lib/server/settings.js');
+            saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
+            const { queueCreateServer } =
+                await import('../../src/lib/server/provisioning/instances.js');
+            const other = queueCreateServer({
+                game: 't5',
+                gameMode: 'zombie',
+                mapSlug: 'theater',
+                hostId,
+                libraryId,
+                image,
+                plutoKey: 'second-key'
+            });
+            if (!other.ok) throw new Error(other.message);
+            await advance();
+            expect(await operationFor(other.deploymentId)).not.toBeNull();
+
+            db()
+                .prepare("UPDATE deployment_servers SET restarted_at=? WHERE deployment_id='retry'")
+                .run(new Date().toISOString());
+            const { POST } = await import('../../src/routes/api/campaigns/[id]/resume/+server.js');
+            expect(POST({ params: { id: 'retry' } }).status).toBe(202);
+            await sweep();
+            const next = requestRestart(job.name, { kind: 'campaign', id: 'retry' });
+            expect(next).toMatchObject({ action: 'rebuild', state: 'waiting', submitted_at: null });
+            expect(next.id).not.toBe(failed.id);
+            expect(
+                db()
+                    .prepare('SELECT generation FROM native_provisioning WHERE deployment_id=?')
+                    .get(next.id)
+            ).toEqual({ generation: 2 });
+            await processProvisioning();
+            const staged = db()
+                .prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?')
+                .get(next.id) as { stage_operation_id: string };
+            const { reportHostOperation, getHostOperation } =
+                await import('../../src/lib/server/hosts/index.js');
+            reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' });
+            await processProvisioning();
+            expect(await operationFor(next.id)).toBeNull();
+            expect((await detail(next.id)).stage).toContain('startup capacity');
+            remote.players = 2;
+            await acknowledge(other.deploymentId);
+            await processProvisioning();
+            expect(await operationFor(next.id)).toBeNull();
+            expect((await detail(next.id)).stage).toContain('player');
+            remote.players = 0;
+            await processProvisioning();
+            const activation = (await operationFor(next.id))!;
+            expect(activation, (await detail(next.id)).stage).toMatchObject({
+                kind: 'instance.rebuild',
+                generation: 2,
+                payload: { nextGeneration: 3, stageId: next.id }
+            });
+            const files = getHostOperation(staged.stage_operation_id)!.payload.files as {
+                id: number;
+                path: string;
+            }[];
+            reportHostOperation(hostId, activation.id, {
+                state: 'completed',
+                result: { before: files.map((file) => ({ ...file, contentBase64: null })) }
+            });
+            await processProvisioning();
+            await sweep();
+            expect(
+                db()
+                    .prepare(
+                        "SELECT restart_state,provisioning_id FROM deployment_servers WHERE deployment_id='retry'"
+                    )
+                    .get()
+            ).toEqual({ restart_state: 'done', provisioning_id: next.id });
+            expect((await detail(failed.id)).state).toBe('failed');
+            expect(
+                db()
+                    .prepare(
+                        "SELECT operation_id FROM provisioning_sources WHERE kind='campaign' AND source_id='retry' ORDER BY rowid"
+                    )
+                    .all()
+            ).toEqual([{ operation_id: failed.id }, { operation_id: next.id }]);
+        }
+    );
+
     it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => {
         const job = await queue();
         await advance();

tests/unit/provisioning-history-performance.test.ts

@@ -87,7 +87,7 @@ describe('provisioning reads with retained lifecycle history', () => {
         fleetOperations(now);
         const queries = prepare.mock.calls
             .map(([sql]) => sql)
-            .filter((sql) => sql.includes('server_deployments'));
+            .filter((sql) => sql.includes('server_deployments') && sql.includes('UNION'));
         prepare.mockRestore();
         expect(queries).toHaveLength(1);
         const statement = handle.prepare(queries[0]);
@@ -98,7 +98,8 @@ describe('provisioning reads with retained lifecycle history', () => {
             'state',
             'stage',
             'error',
-            'started_at'
+            'started_at',
+            'evidence_at'
         ]);
         const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all() as {
             detail: string;

tests/unit/ui-fleet-attention.test.ts

@@ -0,0 +1,90 @@
+import { expect, it, vi } from 'vitest';
+import { render } from 'svelte/server';
+import { writable } from 'svelte/store';
+import type { ComponentProps } from 'svelte';
+import Page from '../../src/routes/+page.svelte';
+import { syncServerTime } from '../../src/lib/ui/stores/servers.js';
+import type { FleetServer } from '../../src/lib/fleet-overview.js';
+
+vi.mock('$app/navigation', () => ({
+    afterNavigate: vi.fn(),
+    replaceState: vi.fn(),
+    goto: vi.fn(),
+    beforeNavigate: vi.fn()
+}));
+vi.mock('$app/stores', () => ({
+    page: writable({ url: new URL('http://localhost/'), data: { user: { role: 'viewer' } } })
+}));
+
+const now = Date.parse('2026-10-19T22:46:00.000Z');
+const healthAt = new Date(now - 8000).toISOString();
+const failedAt = '2026-10-19T18:48:42.000Z';
+const server: FleetServer = {
+    name: 't5_zombie_sumpf_1',
+    game: 't5',
+    map: 'zombie_sumpf',
+    last_map: 'zombie_sumpf',
+    last_players: 0,
+    last_gametype: 'zom',
+    last_seen_at: healthAt,
+    last_checked_at: healthAt,
+    container_state: 'running',
+    container_checked_at: healthAt,
+    last_error: null,
+    roster: [],
+    health: 'online',
+    held: [],
+    active_since: null,
+    inactive_since: null,
+    game_started_at: null,
+    game_ended_at: null
+};
+
+it.each([true, false])(
+    'dates the displayed attention reason when the server is healthy=%s',
+    (healthy) => {
+        syncServerTime(now);
+        const data = {
+            user: null,
+            setupRequired: false,
+            fleetSnapshot: {
+                serverNow: now,
+                hostAttention: 0,
+                reservedNames: [],
+                servers: [
+                    healthy ? server : { ...server, health: 'offline', container_state: 'stopped' }
+                ],
+                integrations: {
+                    native: true,
+                    hostsReady: 1,
+                    hostsTotal: 1,
+                    rcon: true,
+                    rconTimeoutMs: 2000,
+                    defaultRconHost: 'localhost'
+                },
+                operations: [
+                    {
+                        key: 'campaign:one',
+                        group: 'campaign:one',
+                        kind: 'campaign',
+                        server: server.name,
+                        label: 'Update needs attention',
+                        detail: 'Download timed out',
+                        href: '/campaigns/one',
+                        state: 'attention',
+                        startedAt: '2026-10-19T18:38:00.000Z',
+                        evidenceAt: failedAt
+                    }
+                ]
+            }
+            // Rendering needs the public Fleet fields, without the registry's storage metadata.
+        } as unknown as ComponentProps<typeof Page>['data'];
+        const body = render(Page, { props: { data } }).body;
+        const evidence = body.match(/class="evidence-time[^"]*" title="([^"]+)"/)?.[1];
+        expect(evidence).toBe(healthy ? failedAt : healthAt);
+        if (healthy) {
+            expect(body).toContain('Update needs attention');
+            expect(body).toContain('Evidence 3h 57m ago');
+        }
+    }
+);