Complete provisioning job detail pages and lifecycle evidence

tests/unit/create-server.test.ts

@@ -502,7 +502,10 @@ describe('server creation map selection', () => {
         expect(komodoDestroyDeployment).not.toHaveBeenCalled();
         const detail = await (await GET({ params: { id: body.deploymentId } })).json();
         expect(detail.deployment).toMatchObject({ server: serverName, action: 'destroy', state: 'waiting' });
-        expect(detail.events).toHaveLength(2);
+        expect(detail.events.map((event: { message: string }) => event.message)).toEqual(expect.arrayContaining([
+            'Server destruction requested', 'Waiting to verify the server is empty'
+        ]));
+        expect(detail.events.every((event: { id: number }, index: number) => index === 0 || detail.events[index - 1].id < event.id)).toBe(true);
         const missing = await DELETE({ params: { name: 'missing-server' } });
         expect(missing.status).toBe(404);
     });

tests/unit/provisioning-detail.test.ts

@@ -0,0 +1,204 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); });
+afterEach(async () => { await fixture.cleanup(); vi.restoreAllMocks(); });
+
+describe('provisioning detail projection', () => {
+    it('records identical structured and legacy stage text once while preserving changed observations', async () => {
+        const { beginDeployment, deploymentStage, getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const id = beginDeployment('server', 'restart');
+        const update = { phase: 'eligibility' as const,
+            blocker: { code: 'players', message: 'Waiting for 3 players', automatic: true } };
+        deploymentStage(id, update.blocker.message, update);
+        deploymentStage(id, update.blocker.message, update);
+        expect(getDeployment(id)!.events.filter(event => event.message === update.blocker.message)).toHaveLength(1);
+        deploymentStage(id, 'Waiting for 2 players', { ...update, blocker: { ...update.blocker, message: 'Waiting for 2 players' } });
+        const events = getDeployment(id)!.events;
+        expect(events.filter(event => event.message === 'Waiting for 2 players')).toHaveLength(1);
+        expect(events.filter(event => event.kind === 'blocker')).toHaveLength(1);
+    });
+
+    it('keeps legacy full activity while the page and opt-in API return a bounded snapshot without probing', async () => {
+        const history = await import('../../src/lib/server/provisioning/history.js');
+        const komodo = await import('../../src/lib/server/integrations/komodo.js');
+        const rcon = await import('../../src/lib/server/integrations/rcon.js');
+        const inspect = vi.spyOn(komodo, 'komodoInspectHealthy');
+        const status = vi.spyOn(rcon, 'rconStatusResult');
+        const id = history.beginDeployment('t5_zombie_theater_1', 'create');
+        for (let index = 0; index < 40; index++) history.recordDeploymentEvent(id, `Recorded activity ${index}`);
+        history.finishDeployment(id, { ok: false, message: 'A recorded startup failure' });
+        for (let index = 0; index < 8; index++) history.recordDeploymentEvent(id, `Later observation ${index}`);
+
+        const full = history.getDeployment(id)!;
+        const recent = history.getDeployment(id, { events: 'recent' })!;
+        expect(full.events).toHaveLength(50);
+        expect(recent.events).toEqual(full.events.slice(-5));
+        expect(recent.activity).toMatchObject({ total: 50, latestId: full.events.at(-1)!.id,
+            failureExcerpt: { message: 'Failed: A recorded startup failure', kind: 'outcome' } });
+        expect(recent.progress?.phase).toBe('requested');
+        expect(recent.progress?.checkpoints.map(point => point.key)).toEqual(['requested']);
+        const { GET } = await import('../../src/routes/api/provisioning/[id]/+server.js');
+        expect((await (await GET({ params: { id } })).json()).events).toHaveLength(50);
+        expect((await (await GET({ params: { id }, url: new URL('http://localhost/?events=recent') })).json()).events).toHaveLength(5);
+        const { load } = await import('../../src/routes/provisioning/[id]/+page.server.js');
+        expect(load({ params: { id } })?.detail.events).toHaveLength(5);
+        expect(inspect).not.toHaveBeenCalled();
+        expect(status).not.toHaveBeenCalled();
+    });
+
+    it('retains missing sources, normalizes malformed warnings, and leaves legacy evidence unrecorded', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        db().prepare(`INSERT INTO server_deployments (id,server,action,state,stage,started_at,triggered_by,warnings)
+            VALUES ('legacy','old-server','restart','done','Old custom stage','2026-01-01','ui',?)`).run('["Check files",7,null,{"bad":true}]');
+        db().prepare("INSERT INTO provisioning_sources VALUES ('legacy','campaign','missing','old-server')").run();
+        db().prepare("INSERT INTO provisioning_sources VALUES ('legacy','watchdog','incident','old-server')").run();
+        db().prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at)
+            VALUES ('incident','old-server','health','resolved','Recovered game service','2026-01-01','2026-01-01')`).run();
+        const detail = getDeployment('legacy')!;
+        expect(detail.progress).toBeNull();
+        expect(detail.events).toEqual([]);
+        expect(detail.activity).toEqual({ total: 0, latestId: null, failureExcerpt: null });
+        expect(detail.warningMessages).toEqual(['Check files', 'Some recorded warnings could not be read.']);
+        expect(detail.sources).toEqual([
+            { kind: 'campaign', source_id: 'missing', server: 'old-server', label: 'Campaign missing', href: null },
+            { kind: 'watchdog', source_id: 'incident', server: 'old-server', label: 'Recovered game service', href: '/maintenance/incident' }
+        ]);
+        db().prepare("UPDATE server_deployments SET warnings = 'broken json' WHERE id = 'legacy'").run();
+        expect(getDeployment('legacy')!.warningMessages).toEqual(['Recorded warnings could not be read.']);
+    });
+
+    it('correlates reservations by operation ID without assigning a later same-name slot to historical work', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { beginDeployment, getDeployment, finishDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        db().prepare("INSERT INTO servers (name,komodo_name) VALUES ('server','server')").run();
+        const old = beginDeployment('server', 'restart');
+        finishDeployment(old, { ok: false, message: 'Timeout' });
+        const current = beginDeployment('server', 'restart');
+        db().prepare("INSERT INTO restart_rollout (server,requested_at,operation_id) VALUES ('server','2026-01-01',?)").run(current);
+        expect(getDeployment(old)!.capacity.pools[0]).toMatchObject({ kind: 'restart', used: 1, ownsSlot: false,
+            holders: [{ id: current, server: 'server' }] });
+        expect(getDeployment(old)!.progress!.recovery).toBeNull();
+        expect(getDeployment(current)!.capacity.pools[0].ownsSlot).toBe(true);
+        db().prepare('UPDATE restart_rollout SET healthy_at = ?').run(new Date().toISOString());
+        expect(getDeployment(old)!.capacity.pools[0].used).toBe(0);
+        expect(getDeployment(old)!.deployment.state).toBe('failed');
+    });
+
+    it('labels submission order independently of startup reservations and retains queued recipe status', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { beginDeployment, getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const { recordProgress } = await import('../../src/lib/server/provisioning/progress.js');
+        const first = beginDeployment('first', 'create');
+        const next = beginDeployment('next', 'create');
+        db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(next);
+        recordProgress(next, { phase: 'startup-queue', recipeStatus: 'queued', observedAt: null });
+        const detail = getDeployment(next)!;
+        expect(detail.capacity.pools[0]).toMatchObject({ kind: 'startup', used: 1, requestOrder: 1, ownsSlot: false,
+            holders: [{ id: first, server: 'first' }] });
+        expect(detail.progress).toMatchObject({ recipeStatus: 'queued', observedAt: null });
+    });
+
+    it('preserves a failure outcome and evidence before an observation-only same-ID resume', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const history = await import('../../src/lib/server/provisioning/history.js');
+        db().prepare("INSERT INTO servers (name,komodo_name,game_port) VALUES ('server','server',28960)").run();
+        const id = history.beginDeployment('server', 'create');
+        db().prepare("INSERT INTO server_creation_jobs VALUES (?, 'server',28960,'zombie','theater','zombie_theater')").run(id);
+        history.deploymentStage(id, 'Waiting for game evidence', { phase: 'verify-game', checks: [
+            { key: 'game', label: 'Game status', result: 'waiting', source: null }
+        ] });
+        history.finishDeployment(id, { ok: false, message: 'Observation interrupted' });
+        const failed = history.getDeployment(id)!;
+        expect(failed.progress?.phase).toBe('verify-game');
+        expect(history.resumeVerification(id)).toBe(true);
+        const resumed = history.getDeployment(id)!;
+        expect(resumed.deployment).toMatchObject({ id, state: 'awaiting-health', finished_at: null, error: null });
+        expect(resumed.progress).toMatchObject({ phase: 'verify-container', checks: [], observedAt: null });
+        const outcomes = resumed.events.filter(event => event.kind === 'outcome');
+        expect(outcomes).toHaveLength(1);
+        expect(JSON.parse(outcomes[0].details_json!)).toMatchObject({ state: 'failed', finishedAt: failed.deployment.finished_at,
+            error: 'Observation interrupted' });
+        expect(resumed.events.some(event => event.kind === 'resumed')).toBe(true);
+    });
+
+    it('preserves a legacy cleanup failure and keeps deleted cleanup capability authoritative', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { getDeployment, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js');
+        db().prepare(`INSERT INTO server_deployments (id,server,server_deleted_at,action,state,stage,started_at,finished_at,triggered_by,error)
+            VALUES ('cleanup','old-server','2026-01-02','destroy','interrupted',?,'2026-01-01','2026-01-02','ui','App restarted')`).run(REMOVING_SERVER);
+        expect(getDeployment('cleanup')).toMatchObject({ canResumeCleanup: true, deployment: { server_exists: 0 } });
+        expect(resumeCleanup('cleanup')).toBe(true);
+        const result = getDeployment('cleanup')!;
+        expect(result.deployment).toMatchObject({ state: 'waiting', server_deleted_at: '2026-01-02' });
+        const outcome = result.events.find(event => event.kind === 'outcome')!;
+        expect(JSON.parse(outcome.details_json!)).toMatchObject({ state: 'interrupted', finishedAt: '2026-01-02', error: 'App restarted' });
+        expect(result.progress?.phase).toBe('cleanup');
+    });
+});
+
+describe('provisioning activity pagination', () => {
+    it('keeps polling payloads bounded for a retained history with thousands of events', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { beginDeployment, getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const id = beginDeployment('server', 'restart');
+        const insert = db().prepare('INSERT INTO server_deployment_events (deployment_id,created_at,message) VALUES (?,?,?)');
+        db().transaction(() => {
+            for (let index = 0; index < 5000; index++) insert.run(id, '2026-09-28T12:00:00Z', `Observation ${index} ${'x'.repeat(250)}`);
+        })();
+        const full = getDeployment(id)!;
+        const recent = getDeployment(id, { events: 'recent' })!;
+        const fullBytes = Buffer.byteLength(JSON.stringify(full));
+        const recentBytes = Buffer.byteLength(JSON.stringify(recent));
+        expect(recent.activity.total).toBe(5001);
+        expect(recent.events).toHaveLength(5);
+        expect(recentBytes).toBeLessThan(fullBytes / 100);
+        if (process.env.PROVISIONING_BENCHMARK) {
+            const start = performance.now();
+            for (let index = 0; index < 30; index++) getDeployment(id, { events: 'recent' });
+            console.info({ events: full.events.length, fullBytes, recentBytes, meanRecentReadMs: (performance.now() - start) / 30 });
+        }
+    });
+
+    it.each(['newest', 'oldest'] as const)('keeps stable %s pages under concurrent appends and searches literal text', async order => {
+        const { beginDeployment, recordDeploymentEvent } = await import('../../src/lib/server/provisioning/history.js');
+        const { deploymentActivity } = await import('../../src/lib/server/provisioning/activity.js');
+        const id = beginDeployment('server', 'stop');
+        const other = beginDeployment('other', 'stop');
+        for (let index = 0; index < 67; index++) {
+            recordDeploymentEvent(id, index === 30 ? 'literal %_needle' : `Event ${index}`);
+            recordDeploymentEvent(other, 'Unrelated event');
+        }
+        let page = deploymentActivity(id, { order });
+        const watermark = page.watermark;
+        const seen = [...page.events];
+        expect(page.events).toHaveLength(25);
+        recordDeploymentEvent(id, 'New event during reading');
+        while (page.nextCursor !== null) {
+            page = deploymentActivity(id, { order, watermark, cursor: page.nextCursor });
+            seen.push(...page.events);
+        }
+        expect(seen).toHaveLength(68);
+        expect(new Set(seen.map(event => event.id)).size).toBe(68);
+        expect(seen.every(event => event.id <= watermark)).toBe(true);
+        expect(page.total).toBe(68);
+        expect(page.totalEvents).toBe(69);
+        expect(deploymentActivity(id, { query: '%_NEEDLE' }).events.map(event => event.message)).toEqual(['literal %_needle']);
+        expect(deploymentActivity(id, { query: 'not found' }).total).toBe(0);
+    });
+
+    it('validates cursor/search requests and caps page size without changing default legacy responses', async () => {
+        const { beginDeployment, recordDeploymentEvent } = await import('../../src/lib/server/provisioning/history.js');
+        const { GET } = await import('../../src/routes/api/provisioning/[id]/events/+server.js');
+        const id = beginDeployment('server', 'stop');
+        for (let index = 0; index < 130; index++) recordDeploymentEvent(id, `Event ${index}`);
+        const response = GET({ params: { id }, url: new URL('http://localhost/?limit=999') });
+        expect((await response.json()).events).toHaveLength(100);
+        for (const query of ['cursor=-1', 'watermark=NaN', 'limit=0', 'order=invalid', 'cursor=9007199254740992']) {
+            expect(() => GET({ params: { id }, url: new URL(`http://localhost/?${query}`) })).toThrow();
+        }
+        expect(() => GET({ params: { id: 'missing' }, url: new URL('http://localhost/') })).toThrow();
+    });
+});

tests/unit/provisioning-evidence.test.ts

@@ -0,0 +1,284 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { join } from 'node:path';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { gameEvent } from '../helpers/telemetry.js';
+
+vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
+    komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(),
+    komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
+    komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn()
+}));
+vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() }));
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
+
+let fixture: Fixture;
+const server = 't5_zombie_theater_1';
+const status = (map = 'zombie_theater') => ({ ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
+    status: { players: 0, map, gametype: 'zom', roster: [] } });
+
+beforeEach(async () => {
+    vi.resetModules(); vi.resetAllMocks();
+    vi.useFakeTimers({ toFake: ['Date'] });
+    vi.setSystemTime(new Date('2026-09-28T12:00:00Z'));
+    fixture = await newFixture([server]);
+    for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
+        KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
+        SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
+    const komodo = await import('../../src/lib/server/integrations/komodo.js');
+    for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) {
+        vi.mocked(komodo[name]).mockResolvedValue({ ok: true });
+    }
+    vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
+    vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
+    vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
+    const rcon = await import('../../src/lib/server/integrations/rcon.js');
+    vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
+    vi.mocked(rcon.rconStatusResult).mockResolvedValue(status());
+});
+afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); });
+
+async function register() {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    db().prepare(`INSERT INTO servers (name,komodo_type,komodo_name,game,map,game_mode,map_slug,game_port,host_dir)
+        VALUES (?, 'deployment', ?, 't5','zombie_theater','zombie','theater',28960,?)`).run(server, server, join(fixture.t5, server));
+    const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
+    saveServerKey(server, 'saved-key');
+}
+async function detail(id: string) {
+    const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+    return getDeployment(id)!;
+}
+async function telemetry(players: number, map = 'zombie_theater') {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const receivedAt = new Date().toISOString();
+    db().prepare(`INSERT INTO server_telemetry (server,round,players,received_at,payload) VALUES (?,1,?,?,?)
+        ON CONFLICT(server) DO UPDATE SET players=excluded.players,received_at=excluded.received_at,payload=excluded.payload`)
+        .run(server, players, receivedAt, JSON.stringify({ ...gameEvent('heartbeat', 1, players, server, map), receivedAt }));
+    return receivedAt;
+}
+
+describe('provisioning decision evidence', () => {
+    it('distinguishes queued recipes, attempted files, transient map checks, and telemetry completion without replaying startup', async () => {
+        const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
+        const { komodoCreateDeployment, komodoDeploy, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
+        const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
+        const job = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'saved-key' });
+        if (!job.ok) throw new Error(job.message);
+        expect((await detail(job.deploymentId)).progress).toMatchObject({ phase: 'startup-queue', recipeStatus: 'queued', observedAt: null });
+
+        const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { script } = createScript({ name: 'New promotion', targetRelPath: 'promoted.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('promoted while queued') });
+        db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(getLatestVersion(script.id)!.id, script.id);
+        await processCreations();
+        expect((await detail(job.deploymentId)).progress?.recipeStatus).toBe('attempted');
+        expect((await detail(job.deploymentId)).recipe?.files.some(file => file.path === 'promoted.gsc')).toBe(true);
+
+        vi.mocked(rconStatusResult).mockResolvedValueOnce(status('zombie_moon'));
+        vi.setSystemTime(Date.now() + 1000);
+        await processCreations();
+        expect((await detail(job.deploymentId)).progress).toMatchObject({ phase: 'verify-map', blocker: { code: 'map-mismatch' },
+            checks: [{ key: 'container', result: 'pass' }, { key: 'game', result: 'pass' }, { key: 'map', result: 'waiting', expected: 'zombie_theater', actual: 'zombie_moon' }] });
+        vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'container recovering' });
+        await processCreations();
+        expect((await detail(job.deploymentId)).progress?.checks).toEqual([expect.objectContaining({ key: 'container', result: 'waiting' })]);
+
+        vi.setSystemTime(Date.now() + 1000);
+        const receivedAt = await telemetry(0);
+        await processCreations();
+        const completed = await detail(job.deploymentId);
+        expect(completed.deployment.state).toBe('done');
+        expect(completed.progress?.checks).toContainEqual(expect.objectContaining({ key: 'game', source: 'telemetry', result: 'pass', receivedAt }));
+        expect(completed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'verify-map' }));
+        expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
+        expect(komodoDeploy).toHaveBeenCalledTimes(1);
+    });
+
+    it('keeps an uncertain failed request immutable while observing and releasing its exact recovery slot', async () => {
+        await register();
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'response lost' });
+        const job = requestLifecycle(server, 'restart');
+        if (!job.ok) throw new Error(job.error);
+        await processProvisioning();
+        const failed = await detail(job.deploymentId);
+        expect(failed.progress).toMatchObject({ command: { state: 'uncertain' }, recovery: { state: 'held', operationId: job.deploymentId } });
+        const failureEvents = failed.events.filter(event => event.kind === 'outcome');
+        vi.setSystemTime(Date.now() + 15_000);
+        await telemetry(0);
+        await processProvisioning();
+        const recovered = await detail(job.deploymentId);
+        expect(recovered.deployment).toMatchObject({ state: 'failed', finished_at: failed.deployment.finished_at });
+        expect(recovered.progress).toMatchObject({ command: { state: 'uncertain' }, recovery: { state: 'released', operationId: job.deploymentId } });
+        expect(recovered.events.filter(event => event.kind === 'outcome')).toEqual(failureEvents);
+        expect(komodoRestart).toHaveBeenCalledTimes(1);
+    });
+
+    it('records occupied and unknown evidence without treating either as zero or resetting eligibility timing', async () => {
+        await register();
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        const job = requestLifecycle(server, 'restart');
+        if (!job.ok) throw new Error(job.error);
+        vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(null);
+        await processProvisioning();
+        const unknown = await detail(job.deploymentId);
+        expect(unknown.progress?.checks).toEqual([expect.objectContaining({ key: 'occupancy', actual: null, source: null, result: 'unknown' })]);
+        vi.setSystemTime(Date.now() + 1000);
+        await telemetry(3);
+        await processProvisioning();
+        expect((await detail(job.deploymentId)).progress).toMatchObject({ phaseStartedAt: unknown.progress?.phaseStartedAt,
+            blocker: { code: 'players-present' }, checks: [{ key: 'occupancy', actual: 3, source: 'telemetry' }] });
+        vi.setSystemTime(Date.now() + 1000);
+        await telemetry(2);
+        await processProvisioning();
+        expect((await detail(job.deploymentId)).progress?.phaseStartedAt).toBe(unknown.progress?.phaseStartedAt);
+        expect(komodoRestart).not.toHaveBeenCalled();
+    });
+
+    it('preserves failed cleanup evidence through same-ID resume without another destructive command', async () => {
+        await register();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
+        const { resumeCleanup } = await import('../../src/lib/server/provisioning/history.js');
+        const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
+        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
+        db().exec("CREATE TRIGGER fail_cleanup BEFORE DELETE ON servers BEGIN SELECT RAISE(ABORT, 'cleanup interrupted'); END");
+        const job = destroyServer(server);
+        if (!job.ok || !job.deploymentId) throw new Error('Could not queue removal');
+        await processDestructions();
+        const failed = await detail(job.deploymentId);
+        expect(failed.deployment.state).toBe('failed');
+        expect(failed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'delete-deployment' }));
+        expect(failed.progress?.checkpoints.some(point => point.key === 'cleanup')).toBe(false);
+        const failure = failed.events.find(event => event.kind === 'outcome')!;
+        db().exec('DROP TRIGGER fail_cleanup');
+        vi.setSystemTime(Date.now() + 1000);
+        expect(resumeCleanup(job.deploymentId)).toBe(true);
+        await processDestructions();
+        const completed = await detail(job.deploymentId);
+        expect(completed.deployment).toMatchObject({ id: job.deploymentId, state: 'done', server_exists: 0 });
+        expect(completed.events).toContainEqual(failure);
+        expect(completed.events.some(event => event.kind === 'resumed')).toBe(true);
+        expect(completed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'cleanup' }));
+        expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
+        expect(komodoDeleteDeployment).toHaveBeenCalledTimes(1);
+    });
+
+    it('links the actual removal dependency while later eligible creation bypasses it', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
+        const input = { gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'saved-key' };
+        const first = queueCreateServer(input), second = queueCreateServer(input);
+        if (!first.ok || !second.ok) throw new Error('Could not queue creation');
+        const removal = beginDeployment('old-instance', 'destroy');
+        db().prepare('INSERT INTO server_creation_dependencies VALUES (?, ?)').run(first.deploymentId, removal);
+        await processCreations();
+        expect((await detail(first.deploymentId)).progress?.blocker).toMatchObject({ code: 'removal-dependency', href: `/provisioning/${removal}` });
+        expect((await detail(first.deploymentId)).deployment.state).toBe('queued');
+        expect((await detail(second.deploymentId)).deployment.state).toBe('awaiting-health');
+    });
+
+    it.each(['create', 'delete'] as const)('records uncertainty when the %s reply throws without inventing a successful checkpoint', async command => {
+        const komodo = await import('../../src/lib/server/integrations/komodo.js');
+        const { queueCreateServer, processCreations, destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
+        let id: string;
+        if (command === 'create') {
+            vi.mocked(komodo.komodoCreateDeployment).mockRejectedValueOnce(new Error('reply lost'));
+            const job = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'saved-key' });
+            if (!job.ok) throw new Error(job.message);
+            id = job.deploymentId;
+            await processCreations();
+            expect(komodo.komodoCreateDeployment).toHaveBeenCalledTimes(1);
+        } else {
+            await register();
+            vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
+            vi.mocked(komodo.komodoDeleteDeployment).mockRejectedValueOnce(new Error('reply lost'));
+            const job = destroyServer(server);
+            if (!job.ok || !job.deploymentId) throw new Error('Could not queue removal');
+            id = job.deploymentId;
+            await processDestructions();
+            expect(komodo.komodoDeleteDeployment).toHaveBeenCalledTimes(1);
+        }
+        const failed = await detail(id);
+        expect(failed.deployment.state).toBe('failed');
+        expect(failed.progress?.command).toMatchObject({ state: 'uncertain', message: expect.stringContaining('reply lost') });
+        expect(failed.progress?.checkpoints.some(point => point.key === (command === 'create' ? 'create-deployment' : 'delete-deployment'))).toBe(false);
+    });
+
+    it.each(['destroy', 'rebuild'] as const)('retains the resumable %s checkpoint when a beta arrives after confirmed remote deletion', async action => {
+        await register();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
+        const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        const { resumeCleanup, resumeRebuild, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js');
+        const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
+        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
+        vi.mocked(komodoDeleteDeployment).mockImplementationOnce(async () => {
+            const { script, version } = createScript({ name: 'Late beta', targetRelPath: 'beta.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta') });
+            const beta = db().prepare("INSERT INTO betas (script_id,script_version_id,trigger_kind,state) VALUES (?,?,'manual','running')").run(script.id, version.id).lastInsertRowid;
+            db().prepare("INSERT INTO beta_servers (beta_id,server,had_original,stage) VALUES (?,?,0,'ready')").run(beta, server);
+            return { ok: true };
+        });
+        const job = action === 'destroy' ? destroyServer(server) : requestLifecycle(server, 'rebuild');
+        if (!job.ok || !job.deploymentId) throw new Error('Could not queue removal');
+        await (action === 'destroy' ? processDestructions() : processProvisioning());
+        const failed = await detail(job.deploymentId);
+        expect(failed.deployment).toMatchObject({ state: 'failed', stage: REMOVING_SERVER });
+        expect(failed.progress?.checkpoints).toContainEqual(expect.objectContaining({ key: 'delete-deployment' }));
+        expect(action === 'destroy' ? failed.canResumeCleanup : failed.canResumeRebuild).toBe(true);
+        db().prepare("UPDATE betas SET state = 'rolled-back'").run();
+        expect(action === 'destroy' ? resumeCleanup(job.deploymentId) : resumeRebuild(job.deploymentId)).toBe(true);
+        await (action === 'destroy' ? processDestructions() : processProvisioning());
+        expect((await detail(job.deploymentId)).deployment.state).toBe(action === 'destroy' ? 'done' : 'queued');
+        expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
+        expect(komodoDeleteDeployment).toHaveBeenCalledTimes(1);
+    });
+
+    it('ignores occupancy evidence that returns after the request was cancelled', async () => {
+        await register();
+        const { requestLifecycle, cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
+        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        const job = requestLifecycle(server, 'restart');
+        if (!job.ok) throw new Error(job.error);
+        vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
+            expect(cancelProvisioning(job.deploymentId)).toBe(true);
+            return 4;
+        });
+        await processProvisioning();
+        const cancelled = await detail(job.deploymentId);
+        expect(cancelled.deployment.state).toBe('cancelled');
+        expect(cancelled.progress).toMatchObject({ checks: [], blocker: null });
+        expect(komodoRestart).not.toHaveBeenCalled();
+    });
+
+    it('does not attach replacement health to a deleted creation after a delayed probe', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
+        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
+        const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
+        const job = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'saved-key' });
+        if (!job.ok) throw new Error(job.message);
+        await processCreations();
+        vi.mocked(komodoInspectHealthy).mockImplementationOnce(async () => {
+            db().prepare('DELETE FROM servers WHERE name = ?').run(server);
+            await register();
+            return { ok: true, state: 'running' };
+        });
+        await processCreations();
+        const old = await detail(job.deploymentId);
+        expect(old.deployment.server_deleted_at).toBeTruthy();
+        expect(old.progress?.checks).toEqual([]);
+        expect(old.progress?.checkpoints.some(point => point.key.startsWith('verify-'))).toBe(false);
+        expect(rconStatusResult).not.toHaveBeenCalled();
+    });
+});

tests/unit/provisioning-presentation.test.ts

@@ -0,0 +1,131 @@
+import { describe, expect, it } from 'vitest';
+import type { ProvisioningDetail, ProvisioningProgress } from '../../src/lib/provisioning-detail.js';
+import { provisioningActions } from '../../src/lib/provisioning.js';
+import { isProvisioningDetail, isProvisioningEventPage, provisioningGroups, provisioningOutcome, provisioningReturn } from '../../src/lib/ui/provisioning-presentation.js';
+
+const at = '2026-09-28T12:00:00Z';
+function progress(patch: Partial<ProvisioningProgress> = {}): ProvisioningProgress {
+    return { version: 1, phase: 'verify-map', phaseStartedAt: at, observedAt: at, blocker: null, checks: [], checkpoints: [],
+        command: null, recovery: null, recipeStatus: null, ...patch };
+}
+function fixture(): ProvisioningDetail {
+    return {
+        deployment: { id: 'operation-a', server: 't5_zombie_theater_1', action: 'create', state: 'awaiting-health', stage: 'Waiting for a map',
+            server_deleted_at: null, server_exists: 1, started_at: at, finished_at: null, triggered_by: 'ui', error: null, warnings: '[]' },
+        generatedAt: at, events: [], activity: { total: 0, latestId: null, failureExcerpt: null }, sources: [],
+        recipe: null, blueprintName: null, canResumeCleanup: false, canResumeRebuild: false, canResumeVerification: false, canCancelRemoval: false,
+        collection: null, progress: progress(), capacity: { evaluatedAt: at, pools: [] }, warningMessages: []
+    };
+}
+
+describe('provisioning outcome and evidence presentation', () => {
+    it.each(provisioningActions)('describes cancelled %s as terminal, with no automatic execution claim', action => {
+        const detail = fixture();
+        detail.deployment.action = action;
+        detail.deployment.state = 'cancelled';
+        const outcome = provisioningOutcome(detail);
+        expect(outcome.active).toBe(false);
+        expect(outcome.headline).toContain('cancelled');
+        expect(outcome.description).toContain('remaining work will not run');
+        expect(outcome.description).not.toMatch(/verifying|automatically/);
+    });
+
+    it('keeps backend-authorized cleanup actionable and active after identity deletion', () => {
+        const detail = fixture();
+        detail.deployment.action = 'destroy';
+        detail.deployment.state = 'failed';
+        detail.deployment.server_deleted_at = at;
+        detail.deployment.server_exists = 0;
+        detail.canResumeCleanup = true;
+        expect(provisioningOutcome(detail).description).toContain('Resume the remaining local cleanup');
+        detail.deployment.state = 'waiting';
+        detail.canResumeCleanup = false;
+        detail.progress = progress({ phase: 'cleanup', checks: [] });
+        expect(provisioningOutcome(detail).active).toBe(true);
+        expect(provisioningOutcome(detail).description).toContain('without repeating container removal');
+        expect(provisioningOutcome(detail).description).not.toContain('eligibility');
+    });
+
+    it('does not invent completed phases from a later phase or final outcome', () => {
+        const detail = fixture();
+        const steps = provisioningGroups(detail).flatMap(group => group.steps);
+        expect(steps.find(step => step.key === 'prepare')?.state).toBe('unknown');
+        expect(steps.find(step => step.key === 'verify-map')?.state).toBe('current');
+        detail.deployment.state = 'done';
+        expect(provisioningGroups(detail).flatMap(group => group.steps).every(step => step.state === 'unknown')).toBe(true);
+        detail.progress = null;
+        expect(provisioningGroups(detail)).toEqual([]);
+        expect(provisioningOutcome(detail).description).toContain('evidence was not recorded');
+    });
+
+    it('shows passed container and telemetry game checks while waiting on the engine map', () => {
+        const detail = fixture();
+        detail.progress = progress({ checks: [
+            { key: 'container', label: 'Container readiness', result: 'pass', source: 'komodo', checkedAt: at, receivedAt: null, expected: 'healthy', actual: 'healthy', message: null },
+            { key: 'game', label: 'Game status', result: 'pass', source: 'telemetry', checkedAt: at, receivedAt: at, expected: null, actual: null, message: 'Fresh telemetry accepted' },
+            { key: 'map', label: 'Engine map', result: 'waiting', source: 'telemetry', checkedAt: at, receivedAt: at, expected: 'zombie_theater', actual: 'zombie_moon', message: null }
+        ] });
+        const steps = provisioningGroups(detail).flatMap(group => group.steps);
+        expect(steps.find(step => step.key === 'verify-container')?.state).toBe('done');
+        expect(steps.find(step => step.key === 'verify-game')?.state).toBe('done');
+        expect(steps.find(step => step.key === 'verify-map')?.state).toBe('waiting');
+        expect(steps.find(step => step.key === 'verify-game')?.label).not.toContain('RCON');
+        detail.deployment.state = 'done';
+        expect(provisioningOutcome(detail).description).not.toContain('RCON');
+    });
+
+    it('newer unknown readiness evidence supersedes an old completion checkpoint', () => {
+        const detail = fixture();
+        detail.progress = progress({ checkpoints: [{ key: 'verify-game', label: 'Game passed', at, phase: 'verify-game' }],
+            checks: [{ key: 'game', label: 'Game status', result: 'unknown', source: null, checkedAt: '2026-09-28T12:01:00Z', receivedAt: null, expected: null, actual: null, message: 'No fresh evidence' }] });
+        expect(provisioningGroups(detail).flatMap(group => group.steps).find(step => step.key === 'verify-game')).toMatchObject({ state: 'unknown', at: '2026-09-28T12:01:00Z' });
+    });
+
+    it('preserves the failed request when later recovery checks pass', () => {
+        const detail = fixture();
+        detail.deployment.action = 'restart'; detail.deployment.state = 'failed';
+        detail.progress = progress({ phase: 'verify-game', recovery: { state: 'released', at, message: 'Recovery verified and capacity released.' },
+            checks: [{ key: 'game', label: 'Game status', result: 'pass', source: 'rcon', checkedAt: at, receivedAt: null, expected: null, actual: 0, message: null }] });
+        expect(provisioningOutcome(detail).headline).toBe('Restart failed');
+        expect(provisioningGroups(detail).flatMap(group => group.steps).find(step => step.key === 'verify-game')?.state).toBe('done');
+    });
+
+    it('keeps removal checkpoints separate from replacement creation', () => {
+        const detail = fixture(); detail.deployment.action = 'rebuild'; detail.deployment.state = 'waiting';
+        detail.progress = progress({ phase: 'cleanup', checkpoints: [{ key: 'delete-deployment', label: 'Remote deployment deleted', at, phase: 'delete-deployment' }] });
+        const groups = provisioningGroups(detail);
+        expect(groups.map(group => group.key)).toEqual(['request', 'remove', 'create', 'verify']);
+        expect(groups.flatMap(group => group.steps).find(step => step.key === 'delete-deployment')?.state).toBe('done');
+        expect(groups.flatMap(group => group.steps).find(step => step.key === 'cleanup')?.state).toBe('waiting');
+        expect(groups.flatMap(group => group.steps).find(step => step.key === 'prepare')?.state).toBe('pending');
+    });
+});
+
+describe('provisioning response and return-context boundaries', () => {
+    it('retains list filters without accepting an external or unrelated return destination', () => {
+        expect(provisioningReturn('/provisioning?view=history&action=restart&server=Lobby+1&limit=50&redirect=https://evil.test').href)
+            .toBe('/provisioning?view=history&action=restart&server=Lobby+1&limit=50');
+        for (const value of ['https://evil.test', '//evil.test', '/\\evil.test', '/\n/evil.test', '/settings', '/provisioning/other']) {
+            expect(provisioningReturn(value).href).toBe('/provisioning');
+        }
+        expect(provisioningReturn('/collections/launches/launch-a').href).toBe('/collections/launches/launch-a');
+    });
+
+    it('rejects incomplete nested snapshots before they replace a working detail', () => {
+        const detail = fixture();
+        expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(true);
+        expect(isProvisioningDetail(detail, 'another-id')).toBe(false);
+        for (const patch of [
+            { progress: undefined }, { progress: {} }, { progress: { ...detail.progress, checks: [null] } },
+            { capacity: { evaluatedAt: at, pools: [{ kind: 'restart', holders: null }] } }, { activity: { total: 0 } },
+            { warningMessages: {} }, { recipe: { files: [] } }, { sources: [{ kind: 'campaign', label: 'Campaign', source_id: 'a', href: 'javascript:alert(1)' }] }
+        ]) expect(isProvisioningDetail({ ...detail, ...patch }, detail.deployment.id)).toBe(false);
+        expect(isProvisioningDetail({ ...detail, progress: { ...detail.progress, command: { state: 'rejected', at, message: 'Rejected before submission' } } }, detail.deployment.id)).toBe(true);
+    });
+
+    it('rejects an invalid history event without dropping the existing history view', () => {
+        const page = { events: [{ id: 1, created_at: at, message: 'Queued' }], total: 1, totalEvents: 1, latestId: 1, watermark: 1, nextCursor: null };
+        expect(isProvisioningEventPage(page)).toBe(true);
+        expect(isProvisioningEventPage({ ...page, events: [{ id: 1, created_at: 'invalid', message: 'Queued' }] })).toBe(false);
+    });
+});

tests/unit/provisioning-progress.test.ts

@@ -0,0 +1,111 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import Database from 'better-sqlite3';
+import { readFile } from 'node:fs/promises';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+let fixture: Fixture;
+beforeEach(async () => {
+    vi.resetModules();
+    fixture = await newFixture([]);
+    vi.useFakeTimers({ toFake: ['Date'] });
+    vi.setSystemTime(new Date('2026-09-28T12:00:00Z'));
+});
+afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); });
+
+async function setup() {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const handle = db();
+    handle.prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
+        VALUES ('operation', 'server', 'create', 'waiting', 'Recorded stage', '2026-09-28T12:00:00Z', 'test')`).run();
+    const { recordProgress, readProgress } = await import('../../src/lib/server/provisioning/progress.js');
+    const read = () => readProgress(handle.prepare('SELECT observation_json FROM server_deployments WHERE id = ?').get('operation') as { observation_json: string });
+    return { handle, recordProgress, read };
+}
+
+describe('provisioning evidence persistence', () => {
+    it('keeps phase duration stable and avoids semantic event spam during repeated occupancy checks', async () => {
+        const { handle, recordProgress, read } = await setup();
+        for (let players = 4; players > 0; players--) {
+            recordProgress('operation', { phase: 'eligibility', blocker: { code: 'occupied', message: `${players} players remain`, automatic: true },
+                checks: [{ key: 'occupancy', label: 'Players', source: 'telemetry', result: 'waiting', actual: players,
+                    receivedAt: new Date().toISOString() }] });
+            vi.setSystemTime(Date.now() + 15_000);
+        }
+        expect(read()).toMatchObject({ phaseStartedAt: '2026-09-28T12:00:00.000Z', observedAt: '2026-09-28T12:00:45.000Z',
+            checks: [expect.objectContaining({ actual: 1, receivedAt: '2026-09-28T12:00:45.000Z' })] });
+        expect(handle.prepare("SELECT count(*) AS count FROM server_deployment_events WHERE kind = 'blocker'").get()).toEqual({ count: 1 });
+        recordProgress('operation', { phase: 'container-action', blocker: null,
+            checkpoint: { key: 'eligibility', label: 'Eligibility checked', phase: 'eligibility' } });
+        expect(read()).toMatchObject({ phaseStartedAt: '2026-09-28T12:01:00.000Z', blocker: null });
+    });
+
+    it('replaces transient checks while retaining confirmed milestones and prior command uncertainty', async () => {
+        const { handle, recordProgress, read } = await setup();
+        recordProgress('operation', { phase: 'verify-game', command: { state: 'uncertain', message: 'Reply was lost' },
+            checkpoint: { key: 'container-action', label: 'Submission attempted', phase: 'container-action' },
+            checks: [{ key: 'container', label: 'Container', source: 'komodo', result: 'pass' },
+                { key: 'game', label: 'Game', source: 'telemetry', result: 'pass' }] });
+        vi.setSystemTime(Date.now() + 15_000);
+        recordProgress('operation', { phase: 'verify-container', checks: [{ key: 'container', label: 'Container', source: 'komodo', result: 'unknown' }] });
+        expect(read()?.checks).toHaveLength(1);
+        expect(read()?.checks[0].result).toBe('unknown');
+        expect(read()?.checkpoints).toHaveLength(1);
+        expect(read()?.command?.state).toBe('uncertain');
+        expect(handle.prepare("SELECT message FROM server_deployment_events WHERE kind = 'command'").get()).toEqual({ message: 'Reply was lost' });
+    });
+
+    it('keeps operation-owned recovery evidence after slot/server removal and does not rewrite the failed outcome', async () => {
+        const { handle, recordProgress, read } = await setup();
+        handle.prepare("UPDATE server_deployments SET state = 'failed', finished_at = '2026-09-28T12:00:00Z' WHERE id = 'operation'").run();
+        recordProgress('operation', { recovery: { state: 'held', message: 'Observing possible submission' } });
+        vi.setSystemTime(Date.now() + 30_000);
+        recordProgress('operation', { recovery: { state: 'released', message: 'Recovery verified' } });
+        recordProgress('operation', { recovery: { state: 'released', message: 'Recovery verified' } });
+        expect(read()?.recovery).toMatchObject({ state: 'released', at: '2026-09-28T12:00:30.000Z' });
+        expect(handle.prepare("SELECT state, finished_at FROM server_deployments WHERE id = 'operation'").get())
+            .toEqual({ state: 'failed', finished_at: '2026-09-28T12:00:00Z' });
+        expect(handle.prepare("SELECT count(*) AS count FROM server_deployment_events WHERE kind = 'recovery'").get()).toEqual({ count: 2 });
+    });
+
+    it('rolls back state, snapshot, and milestone events together with its caller transaction', async () => {
+        const { handle, recordProgress, read } = await setup();
+        expect(() => handle.transaction(() => {
+            handle.prepare("UPDATE server_deployments SET state = 'running' WHERE id = 'operation'").run();
+            recordProgress('operation', { phase: 'cleanup', checkpoint: { key: 'delete-deployment', label: 'Remote deletion confirmed' } });
+            throw new Error('Interrupted transaction');
+        })()).toThrow('Interrupted transaction');
+        expect(read()).toBeNull();
+        expect(handle.prepare("SELECT state FROM server_deployments WHERE id = 'operation'").get()).toEqual({ state: 'waiting' });
+        expect(handle.prepare('SELECT * FROM server_deployment_events').all()).toEqual([]);
+    });
+
+    it('treats unsupported or malformed historical snapshots as unrecorded', async () => {
+        const { readProgress } = await import('../../src/lib/server/provisioning/progress.js');
+        for (const observation_json of [null, '', '[]', '{broken', '{"version":2}', '{"version":1,"checks":"pass"}']) {
+            expect(readProgress({ observation_json })).toBeNull();
+        }
+    });
+
+    it('adds nullable evidence columns without losing historical identity, row order, or events across reopen', async () => {
+        const schema = (await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8'))
+            .replace(',\n    phase_key TEXT,\n    phase_started_at TEXT,\n    observed_at TEXT,\n    observation_json TEXT', '')
+            .replace(',\n    kind TEXT,\n    phase_key TEXT,\n    details_json TEXT', '')
+            .replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'");
+        const old = new Database(fixture.dbPath);
+        old.exec(schema);
+        old.prepare(`INSERT INTO server_deployments (rowid,id,server,server_deleted_at,action,state,stage,started_at,triggered_by)
+            VALUES (42,'legacy','server','2026-09-01','create','done','Recorded result','2026-08-01','test')`).run();
+        old.prepare("INSERT INTO server_deployment_events (id,deployment_id,created_at,message) VALUES (19,'legacy','2026-08-01','Original evidence')").run();
+        old.close();
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        expect(db().prepare("SELECT rowid, server_deleted_at, observation_json FROM server_deployments WHERE id = 'legacy'").get())
+            .toEqual({ rowid: 42, server_deleted_at: '2026-09-01', observation_json: null });
+        const { recordProgress, readProgress } = await import('../../src/lib/server/provisioning/progress.js');
+        recordProgress('legacy', { recovery: { state: 'unknown', message: 'Historical slot release was not recorded' } });
+        closeDb();
+        const saved = db().prepare("SELECT observation_json FROM server_deployments WHERE id = 'legacy'").get() as { observation_json: string };
+        expect(readProgress(saved)?.recovery?.state).toBe('unknown');
+        expect(db().prepare('SELECT id, message FROM server_deployment_events WHERE id = 19').get()).toEqual({ id: 19, message: 'Original evidence' });
+        expect(db().pragma('foreign_key_check')).toEqual([]);
+    });
+});

tests/unit/provisioning-refresh.test.ts

@@ -0,0 +1,119 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { createProvisioningRefresh, type ProvisioningRefreshState } from '../../src/lib/ui/provisioning-refresh.js';
+
+type Detail = { id: string; revision: number };
+const detail = (id: string, revision = 1): Detail => ({ id, revision });
+const response = (id: string, revision = 1) => Response.json(detail(id, revision));
+const controllers: { dispose: () => void }[] = [];
+afterEach(() => { controllers.splice(0).forEach(controller => controller.dispose()); vi.useRealTimers(); });
+function setup(fetcher: typeof fetch) {
+    let state!: ProvisioningRefreshState<Detail>;
+    const onChange = vi.fn(value => { state = value; });
+    const controller = createProvisioningRefresh<Detail>({ fetcher, onChange, timeoutMs: 100,
+        validate: (value, id): value is Detail => !!value && typeof value === 'object' && (value as Detail).id === id,
+        now: () => '2026-09-28T12:00:00Z' });
+    controllers.push(controller);
+    return { ...controller, onChange, state: () => state };
+}
+function deferred() {
+    let resolve!: (value: Response) => void;
+    let reject!: (error: Error) => void;
+    const promise = new Promise<Response>((done, fail) => { resolve = done; reject = fail; });
+    return { promise, resolve, reject };
+}
+
+describe('provisioning detail refresh ownership', () => {
+    it('uses the server snapshot without another initial request and coalesces background reads', async () => {
+        const pending = deferred();
+        const fetcher = vi.fn(() => pending.promise);
+        const page = setup(fetcher);
+        page.setOperation('a', detail('a'), '2026-09-28T11:59:00Z');
+        expect(fetcher).not.toHaveBeenCalled();
+        const first = page.refresh();
+        const second = page.refresh();
+        expect(fetcher).toHaveBeenCalledTimes(1);
+        pending.resolve(response('a', 2));
+        await Promise.all([first, second]);
+        expect(page.state()).toMatchObject({ detail: detail('a', 2), refreshing: false, refreshedAt: '2026-09-28T12:00:00Z' });
+    });
+
+    it('ignores obsolete success and failure across A → B → A, even when transport ignores abort', async () => {
+        const oldA = deferred(), oldB = deferred(), newA = deferred();
+        const fetcher = vi.fn().mockReturnValueOnce(oldA.promise).mockReturnValueOnce(oldB.promise).mockReturnValueOnce(newA.promise);
+        const page = setup(fetcher);
+        page.setOperation('a', detail('a'));
+        const first = page.refresh();
+        page.setOperation('b', detail('b'));
+        const second = page.refresh();
+        page.setOperation('a', detail('a', 3));
+        const third = page.refresh();
+        newA.resolve(response('a', 4));
+        await third;
+        oldA.resolve(response('a', 0));
+        oldB.reject(new Error('Obsolete failure'));
+        await Promise.all([first, second]);
+        expect(page.state()).toMatchObject({ id: 'a', detail: detail('a', 4), loadError: '', refreshing: false });
+    });
+
+    it('invalidates a pending poll and reconciles an accepted mutation before old data can return', async () => {
+        const old = deferred();
+        const fetcher = vi.fn().mockReturnValueOnce(old.promise).mockResolvedValueOnce(Response.json({ ok: true }))
+            .mockResolvedValueOnce(response('a', 3));
+        const page = setup(fetcher);
+        page.setOperation('a', detail('a'));
+        const polling = page.refresh();
+        await page.act('resume', 'Verification resumed.');
+        old.resolve(response('a', 2));
+        await polling;
+        expect(fetcher).toHaveBeenCalledTimes(3);
+        expect(page.state()).toMatchObject({ detail: detail('a', 3), notice: 'Verification resumed.', busy: null, loadError: '', actionError: '' });
+    });
+
+    it('retains accepted-action feedback if reconciliation fails, and action errors survive successful polls', async () => {
+        const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true }))
+            .mockRejectedValueOnce(new Error('Offline'))
+            .mockResolvedValueOnce(Response.json({ error: 'Removal already began.' }, { status: 409 }))
+            .mockResolvedValueOnce(response('a', 2)).mockResolvedValueOnce(response('a', 3));
+        const page = setup(fetcher);
+        page.setOperation('a', detail('a'));
+        await page.act('resume', 'Resume accepted.');
+        expect(page.state()).toMatchObject({ notice: 'Resume accepted.', loadError: 'Offline', actionError: '', detail: detail('a') });
+        await page.act('cancel', 'Cancelled.');
+        await page.refresh();
+        expect(page.state()).toMatchObject({ actionError: 'Removal already began.', notice: '', loadError: '', detail: detail('a', 3) });
+    });
+
+    it('does not duplicate mutations or poll during them and ignores completion after navigation', async () => {
+        const action = deferred();
+        const fetcher = vi.fn(() => action.promise);
+        const page = setup(fetcher);
+        page.setOperation('a', detail('a'));
+        const first = page.act('resume', 'Resumed.');
+        await page.act('resume', 'Resumed twice.');
+        await page.refresh();
+        expect(fetcher).toHaveBeenCalledTimes(1);
+        page.setOperation('b', detail('b'));
+        action.resolve(Response.json({ ok: true }));
+        await first;
+        expect(fetcher).toHaveBeenCalledTimes(1);
+        expect(page.state()).toMatchObject({ id: 'b', busy: null, notice: '', actionError: '' });
+    });
+
+    it('times out reads without discarding the last snapshot and suppresses updates after unmount', async () => {
+        vi.useFakeTimers();
+        const fetcher = vi.fn((_url, init) => new Promise<Response>((_, reject) => {
+            init?.signal?.addEventListener('abort', () => reject(new Error('aborted')));
+        })) as typeof fetch;
+        const page = setup(fetcher);
+        page.setOperation('a', detail('a'));
+        const pending = page.refresh();
+        await vi.advanceTimersByTimeAsync(100);
+        await pending;
+        expect(page.state()).toMatchObject({ detail: detail('a'), loadError: 'Operation refresh timed out. Try again.', refreshing: false });
+        const next = page.refresh();
+        page.dispose();
+        const calls = page.onChange.mock.calls.length;
+        await next;
+        expect(page.onChange).toHaveBeenCalledTimes(calls);
+    });
+});

tests/unit/provisioning.test.ts

@@ -303,7 +303,8 @@ describe('Watchdog provisioning and telemetry evidence', () => {
         expect(result.ok).toBe(true);
         if (!result.ok) throw new Error(result.error);
         expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toEqual(result);
-        expect(getDeployment(result.deploymentId)?.sources).toEqual([{ kind: 'watchdog', source_id: 'incident', server }]);
+        expect(getDeployment(result.deploymentId)?.sources).toEqual([{ kind: 'watchdog', source_id: 'incident', server,
+            label: 'Server down', href: '/maintenance/incident' }]);
         expect(await operation(result.deploymentId)).toMatchObject({ triggered_by: 'watchdog:incident' });
         db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run();
         await processProvisioning();

tests/unit/restart-rollout.test.ts

@@ -348,7 +348,8 @@ describe('Provisioning ownership', () => {
         const source = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?').get('campaign') as { provisioning_id: string };
         const id = source.provisioning_id;
         expect(provisioningOperation(id)).toMatchObject({ action: 'restart', state: 'waiting', submitted_at: null });
-        expect(getDeployment(id)?.sources).toEqual([{ kind: 'campaign', source_id: 'campaign', server: servers[0] }]);
+        expect(getDeployment(id)?.sources).toEqual([{ kind: 'campaign', source_id: 'campaign', server: servers[0],
+            label: 'Promotion campaign', href: '/campaigns/campaign' }]);
         expect(komodoRestart).not.toHaveBeenCalled();
         expect(komodoInspectHealthy).not.toHaveBeenCalled();
         expect(rconGetPlayerCount).not.toHaveBeenCalled();