Complete provisioning job detail pages and lifecycle evidence

src/lib/server/db/index.ts

@@ -243,8 +243,10 @@ function migrateProvisioningActions(handle: Database.Database): void {
         handle.transaction(() => {
             handle.exec(table.sql.replace('server_deployments', 'server_deployments_new')
                 .replace("'create', 'destroy'", "'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'"));
-            handle.exec(`INSERT INTO server_deployments_new (rowid, id, server, action, state, stage, started_at, finished_at, triggered_by, error, warnings)
-                SELECT rowid, id, server, action, state, stage, started_at, finished_at, triggered_by, error, warnings FROM server_deployments;
+            const columns = (handle.prepare('PRAGMA table_info(server_deployments)').all() as { name: string }[])
+                .map(column => `"${column.name.replaceAll('"', '""')}"`).join(', ');
+            handle.exec(`INSERT INTO server_deployments_new (rowid, ${columns})
+                SELECT rowid, ${columns} FROM server_deployments;
                 DROP TABLE server_deployments;
                 ALTER TABLE server_deployments_new RENAME TO server_deployments;`);
             if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Provisioning migration found broken references');
@@ -309,7 +311,16 @@ export function db(): Database.Database {
         migrateTable(handle, 'server_deployments', [
             { name: 'submitted_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' },
             { name: 'checked_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN checked_at TEXT' },
-            { name: 'players', ddl: 'ALTER TABLE server_deployments ADD COLUMN players INTEGER' }
+            { name: 'players', ddl: 'ALTER TABLE server_deployments ADD COLUMN players INTEGER' },
+            { name: 'phase_key', ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_key TEXT' },
+            { name: 'phase_started_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN phase_started_at TEXT' },
+            { name: 'observed_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN observed_at TEXT' },
+            { name: 'observation_json', ddl: 'ALTER TABLE server_deployments ADD COLUMN observation_json TEXT' }
+        ]);
+        migrateTable(handle, 'server_deployment_events', [
+            { name: 'kind', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN kind TEXT' },
+            { name: 'phase_key', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN phase_key TEXT' },
+            { name: 'details_json', ddl: 'ALTER TABLE server_deployment_events ADD COLUMN details_json TEXT' }
         ]);
         for (const table of ['deployment_servers', 'beta_servers', 'maintenance_jobs']) {
             migrateTable(handle, table, [{ name: 'provisioning_id', ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT` }]);

src/lib/server/db/schema.sql

@@ -452,14 +452,21 @@ CREATE TABLE IF NOT EXISTS server_deployments (
     finished_at TEXT,
     triggered_by TEXT NOT NULL,
     error TEXT,
-    warnings TEXT NOT NULL DEFAULT '[]'
+    warnings TEXT NOT NULL DEFAULT '[]',
+    phase_key TEXT,
+    phase_started_at TEXT,
+    observed_at TEXT,
+    observation_json TEXT
 );
 
 CREATE TABLE IF NOT EXISTS server_deployment_events (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     deployment_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE,
     created_at TEXT NOT NULL,
-    message TEXT NOT NULL
+    message TEXT NOT NULL,
+    kind TEXT,
+    phase_key TEXT,
+    details_json TEXT
 );
 CREATE INDEX IF NOT EXISTS idx_server_deployment_events ON server_deployment_events(deployment_id, id);
 

src/lib/server/fleet/status-evidence.ts

@@ -36,30 +36,48 @@ export function telemetryEvidence(server: string, after: string | null = latestG
 
 /** Positive occupancy from either source always vetoes an empty-server claim. */
 export async function probePlayerCount(server: string): Promise<number | null> {
+    return (await probePlayerEvidence(server)).players;
+}
+
+export interface PlayerEvidence {
+    players: number | null;
+    source: 'rcon' | 'telemetry' | null;
+    observedAt: string;
+    receivedAt?: string;
+}
+
+/** The same eligibility decision, with provenance from the probes already performed. */
+export async function probePlayerEvidence(server: string): Promise<PlayerEvidence> {
+    const fromTelemetry = (value: TelemetryEvidence): PlayerEvidence => ({
+        players: value.players, source: 'telemetry', observedAt: new Date().toISOString(), receivedAt: value.receivedAt
+    });
     const occupied = telemetryEvidence(server);
-    if (occupied && occupied.players > 0) return occupied.players;
+    if (occupied && occupied.players > 0) return fromTelemetry(occupied);
     const players = await rconGetPlayerCount(server).catch(() => null);
     // Read after the network request: a player may join while RCON is pending.
     const telemetry = telemetryEvidence(server);
-    if (players !== null && players > 0) return players;
-    if (telemetry && telemetry.players > 0) return telemetry.players;
-    return players === 0 || telemetry?.players === 0 ? 0 : null;
+    if (players !== null && players > 0) return { players, source: 'rcon', observedAt: new Date().toISOString() };
+    if (telemetry && telemetry.players > 0) return fromTelemetry(telemetry);
+    if (players === 0) return { players, source: 'rcon', observedAt: new Date().toISOString() };
+    if (telemetry?.players === 0) return fromTelemetry(telemetry);
+    return { players: null, source: null, observedAt: new Date().toISOString() };
 }
 
 /** Komodo readiness remains a caller prerequisite. Old-process telemetry is excluded. */
 export async function probeServerStatus(server: string, after?: string | null): Promise<
-    { ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry' } | { ok: false; reason: string }
+    { ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry'; observedAt: string; receivedAt?: string } | { ok: false; reason: string; observedAt: string }
 > {
     const current = telemetryEvidence(server, after);
-    if (current) return { ok: true, ...current, source: 'telemetry' };
+    if (current) return { ok: true, ...current, source: 'telemetry', observedAt: new Date().toISOString() };
     let reason = 'RCON unavailable';
     try {
         const rcon = await rconStatusResult(server);
         const latest = telemetryEvidence(server, after);
-        if (latest) return { ok: true, ...latest, source: 'telemetry' };
-        if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon' };
+        if (latest) return { ok: true, ...latest, source: 'telemetry', observedAt: new Date().toISOString() };
+        if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', observedAt: new Date().toISOString() };
         reason = rcon.reason;
     } catch (err) { reason = (err as Error).message; }
     const telemetry = telemetryEvidence(server, after);
-    return telemetry ? { ok: true, ...telemetry, source: 'telemetry' } : { ok: false, reason };
+    return telemetry ? { ok: true, ...telemetry, source: 'telemetry', observedAt: new Date().toISOString() }
+        : { ok: false, reason, observedAt: new Date().toISOString() };
 }

src/lib/server/provisioning/activity.ts

@@ -0,0 +1,56 @@
+import { db } from '../db/index.js';
+import type { ProvisioningEvent, ProvisioningEventPage } from '../../provisioning-detail.js';
+
+type ActivityRow = ProvisioningEvent;
+
+export interface ActivityQuery {
+    order?: 'newest' | 'oldest';
+    query?: string;
+    cursor?: number;
+    watermark?: number;
+    limit?: number;
+}
+
+export function activitySummary(id: string): { total: number; latestId: number | null } {
+    return db().prepare(`SELECT COUNT(*) AS total, MAX(id) AS latestId
+        FROM server_deployment_events WHERE deployment_id = ?`).get(id) as { total: number; latestId: number | null };
+}
+
+/** ID ordering and a fixed upper watermark keep older pages stable as new events arrive. */
+export function deploymentActivity(id: string, options: ActivityQuery = {}): ProvisioningEventPage {
+    const summary = activitySummary(id);
+    const watermark = Math.min(options.watermark ?? summary.latestId ?? 0, summary.latestId ?? 0);
+    const limit = Math.max(1, Math.min(100, Math.trunc(options.limit ?? 25)));
+    const newest = options.order !== 'oldest';
+    const query = options.query ?? '';
+    const base = 'deployment_id = ? AND id <= ? AND instr(lower(message), lower(?)) > 0';
+    const values: (string | number)[] = [id, watermark, query];
+    const total = (db().prepare(`SELECT COUNT(*) AS n FROM server_deployment_events WHERE ${base}`)
+        .get(...values) as { n: number }).n;
+    const cursor = options.cursor === undefined ? '' : ` AND id ${newest ? '<' : '>'} ?`;
+    if (options.cursor !== undefined) values.push(options.cursor);
+    const rows = db().prepare(`SELECT id, created_at, message, kind, phase_key, details_json
+        FROM server_deployment_events WHERE ${base}${cursor}
+        ORDER BY id ${newest ? 'DESC' : 'ASC'} LIMIT ?`).all(...values, limit + 1) as ActivityRow[];
+    const more = rows.length > limit;
+    const events = rows.slice(0, limit);
+    return { events, total, totalEvents: summary.total, latestId: summary.latestId, watermark,
+        nextCursor: more ? events.at(-1)!.id : null };
+}
+
+export function deploymentEvents(id: string, recent = false): ProvisioningEvent[] {
+    if (recent) return (db().prepare(`SELECT id, created_at, message, kind, phase_key, details_json
+        FROM server_deployment_events WHERE deployment_id = ? ORDER BY id DESC LIMIT 5`).all(id) as ActivityRow[]).reverse();
+    return (db().prepare(`SELECT id, created_at, message, kind, phase_key, details_json
+        FROM server_deployment_events WHERE deployment_id = ? ORDER BY id`).all(id) as ActivityRow[]);
+}
+
+export function latestFailureEvent(id: string): ProvisioningEvent | null {
+    const row = db().prepare(`SELECT id, created_at, message, kind, phase_key, details_json
+        FROM server_deployment_events WHERE deployment_id = ?
+        AND (kind IN ('failed', 'interrupted') OR (kind = 'outcome'
+            AND CASE WHEN json_valid(details_json) THEN json_extract(details_json, '$.state') END IN ('failed', 'interrupted'))
+            OR (kind IS NULL AND (message LIKE 'Failed:%' OR message LIKE 'Interrupted by application restart.%')))
+        ORDER BY id DESC LIMIT 1`).get(id) as ActivityRow | undefined;
+    return row ?? null;
+}

src/lib/server/provisioning/history.ts

@@ -2,20 +2,35 @@ import { randomUUID } from 'node:crypto';
 import { db } from '../db/index.js';
 import { recipeForDeployment, summarizeRecipe } from '../fleet/launch-recipes.js';
 import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
+import type { ProvisioningCapacity, ProvisioningDeployment, ProvisioningDetail, ProvisioningSource } from '../../provisioning-detail.js';
+import { fleetSettings } from '../settings.js';
+import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js';
+import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
 
 export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
     const id = randomUUID();
-    db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
-        VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
-    recordDeploymentEvent(id, action === 'create' ? 'Server creation requested'
-        : action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested`);
+    db().transaction(() => {
+        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
+            VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
+        recordProgress(id, { phase: 'requested', observedAt: null, checkpoint: { key: 'requested', phase: 'requested',
+            label: action === 'create' ? 'Server creation requested'
+                : action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested` } });
+    })();
     return id;
 }
 
-export function deploymentStage(id: string, stage: string): void {
-    const previous = db().prepare('SELECT stage FROM server_deployments WHERE id = ?').get(id) as { stage: string } | undefined;
-    db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id);
-    if (previous && previous.stage !== stage) recordDeploymentEvent(id, stage);
+export function deploymentStage(id: string, stage: string, update?: ProgressUpdate): void {
+    db().transaction(() => {
+        const previous = db().prepare('SELECT stage FROM server_deployments WHERE id = ?').get(id) as { stage: string } | undefined;
+        const changed = !!previous && previous.stage !== stage;
+        const before = update && changed ? activitySummary(id).latestId ?? 0 : 0;
+        db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id);
+        if (update) recordProgress(id, update);
+        // Structured transitions may already carry the exact legacy stage text.
+        const alreadyRecorded = update && changed && db().prepare(`SELECT 1 FROM server_deployment_events
+            WHERE deployment_id = ? AND id > ? AND message = ? LIMIT 1`).get(id, before, stage);
+        if (changed && !alreadyRecorded) recordDeploymentEvent(id, stage);
+    })();
 }
 
 export function recordDeploymentEvent(id: string, message: string): void {
@@ -23,19 +38,83 @@ export function recordDeploymentEvent(id: string, message: string): void {
         .run(id, new Date().toISOString(), message);
 }
 
-export function getDeployment(id: string) {
-    const deployment = db().prepare(`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
-        FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id);
-    if (!deployment) return null;
-    const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id);
-    const sources = db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id').all(id);
-    const launch = recipeForDeployment(id);
-    const collection = db().prepare(`SELECT c.id, c.name FROM (
-        SELECT collection_id, deployment_id FROM server_collection_members
-        UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
-        JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ?`).get(id) ?? null;
-    return { deployment, events, sources, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
-        canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection };
+function warningMessages(raw: string): string[] {
+    try {
+        const parsed: unknown = JSON.parse(raw);
+        if (!Array.isArray(parsed)) return ['Recorded warnings could not be read.'];
+        const messages = parsed.filter((value): value is string => typeof value === 'string');
+        if (messages.length !== parsed.length) messages.push('Some recorded warnings could not be read.');
+        return messages;
+    } catch { return ['Recorded warnings could not be read.']; }
+}
+
+function sourceDetails(source: Omit<ProvisioningSource, 'label' | 'href'>): ProvisioningSource {
+    const id = source.source_id;
+    if (source.kind === 'campaign') {
+        const row = db().prepare('SELECT action FROM deployments WHERE id = ?').get(id) as { action: string } | undefined;
+        const action = row?.action === 'promote' ? 'Promotion' : row?.action === 'rollback' ? 'Rollback' : row?.action === 'drift' ? 'Reconciliation' : 'Campaign';
+        return { ...source, label: `${action} ${id.slice(0, 8)}`, href: row ? `/campaigns/${encodeURIComponent(id)}` : null };
+    }
+    if (source.kind === 'beta') {
+        const row = db().prepare('SELECT notes FROM betas WHERE id = ?').get(id) as { notes: string | null } | undefined;
+        const label = row?.notes?.trim().split('\n')[0].slice(0, 120);
+        return { ...source, label: label ? `Beta #${id}: ${label}` : `Beta #${id}`, href: row ? `/betas/${encodeURIComponent(id)}` : null };
+    }
+    if (source.kind === 'watchdog') {
+        const row = db().prepare('SELECT summary FROM watchdog_incidents WHERE id = ?').get(id) as { summary: string } | undefined;
+        return { ...source, label: row?.summary ?? `Maintenance finding ${id.slice(0, 8)}`, href: row ? `/maintenance/${encodeURIComponent(id)}` : null };
+    }
+    const row = db().prepare('SELECT kind FROM maintenance_jobs WHERE id = ?').get(id) as { kind: string } | undefined;
+    return { ...source, label: row && row.kind !== 'legacy' ? `Maintenance ${row.kind} #${id}` : `Maintenance #${id}`,
+        href: row ? `/maintenance?view=activity#job-${encodeURIComponent(id)}` : null };
+}
+
+/** These are current pool snapshots, not a second admission/eligibility engine. */
+function capacityFor(deployment: ProvisioningDeployment, evaluatedAt: string): ProvisioningCapacity {
+    const settings = fleetSettings();
+    const pools: ProvisioningCapacity['pools'] = [];
+    if (deployment.action === 'create' || deployment.action === 'rebuild') {
+        const jobs = db().prepare(`SELECT id, server, state FROM server_deployments
+            WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health')
+            ORDER BY started_at, rowid`).all() as { id: string; server: string; state: string }[];
+        const holders = jobs.filter(job => job.state !== 'queued').map(({ id, server }) => ({ id, server }));
+        const order = jobs.filter(job => job.state === 'queued').findIndex(job => job.id === deployment.id);
+        pools.push({ kind: 'startup', limit: settings.createConcurrency, used: holders.length, holders,
+            requestOrder: order < 0 ? null : order + 1, ownsSlot: holders.some(holder => holder.id === deployment.id) });
+    }
+    if (['start', 'restart', 'rebuild'].includes(deployment.action)) {
+        const holders = db().prepare(`SELECT operation_id AS id, server FROM restart_rollout WHERE healthy_at IS NULL
+            ORDER BY requested_at, server`).all() as { id: string | null; server: string }[];
+        pools.push({ kind: 'restart', limit: settings.restartConcurrency, used: holders.length, holders,
+            requestOrder: null, ownsSlot: holders.some(holder => holder.id === deployment.id) });
+    }
+    return { evaluatedAt, pools };
+}
+
+/** The default retains the full legacy event response. Page clients opt into a bounded window. */
+export function getDeployment(id: string, options: { events?: 'all' | 'recent' } = {}): ProvisioningDetail | null {
+    return db().transaction(() => {
+        const row = db().prepare(`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
+            FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id) as
+            (ProvisioningDeployment & { observation_json: string | null; phase_key: string | null; phase_started_at: string | null; observed_at: string | null }) | undefined;
+        if (!row) return null;
+        const generatedAt = new Date().toISOString();
+        const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
+        const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
+            .all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
+        const launch = recipeForDeployment(id);
+        const collection = db().prepare(`SELECT c.id, c.name FROM (
+            SELECT collection_id, deployment_id FROM server_collection_members
+            UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
+            JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
+        return { deployment, events: deploymentEvents(id, options.events === 'recent'),
+            activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
+            recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
+            canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),
+            canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
+            progress: readProgress(row), capacity: capacityFor(deployment, generatedAt),
+            warningMessages: warningMessages(deployment.warnings), generatedAt };
+    })();
 }
 
 // This stage is written only after Komodo has confirmed deployment deletion.
@@ -48,10 +127,13 @@ export function canCancelRemoval(id: string): boolean {
 /** Competes atomically with the worker's claim, before any external removal. */
 export function cancelRemoval(id: string): boolean {
     return db().transaction(() => {
+        const finishedAt = new Date().toISOString();
         const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
-            WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(new Date().toISOString(), id, REMOVING_SERVER);
+            WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(finishedAt, id, REMOVING_SERVER);
         if (!result.changes) return false;
-        recordDeploymentEvent(id, 'Queued removal cancelled. The server and its files were kept.');
+        recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome',
+            message: 'Queued removal cancelled. The server and its files were kept.',
+            details: { state: 'cancelled', finishedAt } } });
         return true;
     })();
 }
@@ -76,8 +158,10 @@ export function canResumeCleanup(id: string): boolean {
 export function resumeCleanup(id: string): boolean {
     return db().transaction(() => {
         if (!canResumeCleanup(id)) return false;
+        preserveOutcome(id);
         db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
-        recordDeploymentEvent(id, 'Resumed local cleanup after confirmed Komodo deletion');
+        recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
+            event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed Komodo deletion' } });
         return true;
     })();
 }
@@ -100,29 +184,55 @@ export function canResumeRebuild(id: string): boolean {
 }
 
 export function resumeRebuild(id: string): boolean {
-    if (!canResumeRebuild(id)) return false;
-    db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
-    recordDeploymentEvent(id, 'Resumed rebuild cleanup after confirmed Komodo deletion');
-    return true;
+    return db().transaction(() => {
+        if (!canResumeRebuild(id)) return false;
+        preserveOutcome(id);
+        db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
+        recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
+            event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed Komodo deletion' } });
+        return true;
+    })();
 }
 
 /** Resume observation only. Never repeat a container creation or restart. */
 export function resumeVerification(id: string): boolean {
     return db().transaction(() => {
         if (!canResumeVerification(id)) return false;
+        preserveOutcome(id);
         db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
-        recordDeploymentEvent(id, 'Resumed verification of the existing server');
-        deploymentStage(id, 'Waiting for Komodo health');
+        deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null,
+            event: { kind: 'resumed', message: 'Resumed verification of the existing server' } });
         return true;
     })();
 }
 
 export function finishDeployment(id: string, result: { ok: boolean; message?: string; warnings?: string[] }): void {
     const state = !result.ok ? 'failed' : result.warnings?.length ? 'warning' : 'done';
-    db().prepare(`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`)
-        .run(state, new Date().toISOString(), result.message ?? null, JSON.stringify(result.warnings ?? []), id);
-    for (const warning of result.warnings ?? []) recordDeploymentEvent(id, warning);
-    recordDeploymentEvent(id, result.ok ? (state === 'warning' ? 'Completed with warnings' : 'Completed') : `Failed: ${result.message ?? 'Unknown error'}`);
+    db().transaction(() => {
+        const finishedAt = new Date().toISOString();
+        db().prepare(`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`)
+            .run(state, finishedAt, result.message ?? null, JSON.stringify(result.warnings ?? []), id);
+        for (const warning of result.warnings ?? []) recordProgress(id, { event: { kind: 'warning', message: warning } });
+        recordProgress(id, { blocker: null, event: { kind: 'outcome',
+            message: result.ok ? (state === 'warning' ? 'Completed with warnings' : 'Completed') : `Failed: ${result.message ?? 'Unknown error'}`,
+            details: { state, finishedAt, error: result.message ?? null, warnings: result.warnings ?? [] } } });
+    })();
+}
+
+/** Legacy failures also need a durable structured result before same-ID resume clears it. */
+function preserveOutcome(id: string): void {
+    const row = db().prepare('SELECT state, finished_at, error, warnings, observation_json FROM server_deployments WHERE id = ?').get(id) as
+        { state: string; finished_at: string | null; error: string | null; warnings: string; observation_json: string | null } | undefined;
+    if (!row || !['failed', 'interrupted', 'warning', 'done', 'cancelled'].includes(row.state)) return;
+    const previous = db().prepare("SELECT details_json FROM server_deployment_events WHERE deployment_id = ? AND kind = 'outcome' ORDER BY id DESC LIMIT 1")
+        .get(id) as { details_json: string | null } | undefined;
+    try {
+        const details = JSON.parse(previous?.details_json ?? 'null');
+        if (details?.state === row.state && details?.finishedAt === row.finished_at) return;
+    } catch { /* Preserve the row's known outcome when old event metadata is missing. */ }
+    recordProgress(id, { observedAt: readProgress(row)?.observedAt ?? null, event: { kind: 'outcome',
+        message: `${row.state === 'interrupted' ? 'Interrupted' : 'Recorded outcome'}: ${row.error ?? row.state}`,
+        details: { state: row.state, finishedAt: row.finished_at, error: row.error, warnings: warningMessages(row.warnings) } } });
 }
 
 export function listDeployments() {
@@ -132,9 +242,12 @@ export function listDeployments() {
 }
 
 export function interruptDeployments(): void {
-    const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running'").all() as { id: string }[];
-    db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?,
-        error = 'The application restarted before this operation finished. Check the server before retrying.'
-        WHERE state = 'running'`).run(new Date().toISOString());
-    for (const { id } of running) recordDeploymentEvent(id, 'Interrupted by application restart. Check the server before retrying.');
+    db().transaction(() => {
+        const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running'").all() as { id: string }[];
+        const finishedAt = new Date().toISOString();
+        const error = 'The application restarted before this operation finished. Check the server before retrying.';
+        db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running'`).run(finishedAt, error);
+        for (const { id } of running) recordProgress(id, { blocker: null, event: { kind: 'outcome',
+            message: 'Interrupted by application restart. Check the server before retrying.', details: { state: 'interrupted', finishedAt, error } } });
+    })();
 }

src/lib/server/provisioning/index.ts

@@ -4,4 +4,5 @@ export { requestLifecycle, requestRestart, requestWatchdogLifecycle, escalateWat
 export type { LifecycleOperation, RestartSource } from './requests.js';
 export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextAvailablePort, nextLobbyNumber } from './instances.js';
 export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js';
+export { deploymentActivity } from './activity.js';
 export { restartSettling, restartRecoveryPending } from './restarts.js';

src/lib/server/provisioning/instances.ts

@@ -1,5 +1,5 @@
 import { fleetSettings } from '../settings.js';
-import { probePlayerCount, probeServerStatus } from '../fleet/status-evidence.js';
+import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence.js';
 import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
 import { randomUUID } from 'node:crypto';
 import type { LaunchPreview } from '$lib/deployment-plan.js';
@@ -25,6 +25,8 @@ import {
 import { MAP_CONFIG_FILE } from '../fleet/server-config.js';
 import { recordScript } from '../fleet/inventory.js';
 import { getVersion } from '../fleet/scripts.js';
+import type { ProvisioningBlocker } from '../../provisioning-detail.js';
+import { recordProgress, type ProgressUpdate } from './progress.js';
 
 export type CreateServerFailure =
     | 'bad-request'
@@ -147,6 +149,13 @@ function buildSpec(args: {
 }
 
 let creating = false;
+async function observeCommand<T>(operation: string, command: () => Promise<T>): Promise<T> {
+    try { return await command(); }
+    catch (err) {
+        recordProgress(operation, { command: { state: 'uncertain', message: 'No confirmed command result was received: ' + (err as Error).message } });
+        throw err;
+    }
+}
 interface PreparedCreation {
     ok: true; name: string; port: number; plutoKey: string; mapEntry: ZombieMapEntry;
 }
@@ -225,7 +234,8 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
         db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
         db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
             .run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical);
-        deploymentStage(id, 'Queued for server creation');
+        deploymentStage(id, 'Queued for server creation', { phase: 'startup-queue', recipeStatus: 'queued', observedAt: null,
+            blocker: { code: 'startup-queue', message: 'Waiting for startup admission and available capacity.', automatic: true } });
         return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id };
     })();
 }
@@ -252,7 +262,8 @@ export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServ
             .run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
         db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
         db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe));
-        deploymentStage(id, 'Waiting to rebuild the empty server');
+        deploymentStage(id, 'Waiting to rebuild the empty server', { phase: 'eligibility', recipeStatus: 'queued', observedAt: null,
+            blocker: { code: 'eligibility', message: 'Waiting for safe rebuild eligibility checks.', automatic: true } });
         return { ok: true as const, name, port: server.game_port!, deploymentId: id };
     })();
 }
@@ -266,12 +277,15 @@ export async function rebuildServer(id: string): Promise<void> {
     if (blocker) {
         // Keep the cleanup checkpoint intact when resuming after remote deletion.
         if (job.stage === REMOVING_SERVER) finishDeployment(id, { ok: false, message: blocker });
-        else deploymentStage(id, blocker);
+        else deploymentStage(id, blocker, { phase: 'eligibility', blocker: betaRemovalEvidence(job.server, blocker), checks: [] });
         return;
     }
     if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) {
         // The cleanup checkpoint must survive capacity waits and app restarts.
-        if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding');
+        const progress: ProgressUpdate = { phase: job.stage === REMOVING_SERVER ? 'cleanup' : 'eligibility',
+            blocker: { code: 'startup-capacity', message: 'Waiting for a server creation slot before rebuilding', automatic: true } };
+        if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding', progress);
+        else recordProgress(id, progress);
         return;
     }
     // Recheck prerequisites while the original instance still exists.
@@ -290,7 +304,9 @@ export async function rebuildServer(id: string): Promise<void> {
         db().transaction(() => {
             db().prepare('UPDATE server_rebuild_jobs SET removed_at = ? WHERE deployment_id = ?').run(new Date().toISOString(), id);
             db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
-            deploymentStage(id, 'Queued for server creation');
+            deploymentStage(id, 'Queued for server creation', { phase: 'startup-queue', checks: [],
+                blocker: { code: 'startup-queue', message: 'Old instance removed; waiting for fresh-instance startup admission.', automatic: true },
+                checkpoint: { key: 'cleanup', label: 'Old instance files and registry removed' } });
         })();
     } catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
     finally { destroying.delete(job.server); }
@@ -369,11 +385,15 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
     // Resolve once more at provisioning so promotions made while queued also apply.
     // Persist the exact attempted files for history, never as input to future launches.
     const recipe = captureRecipe(game, mapEntry.slug);
-    db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)
-        ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`)
-        .run(operation, JSON.stringify(recipe));
+    db().transaction(() => {
+        db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)
+            ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`)
+            .run(operation, JSON.stringify(recipe));
+        deploymentStage(operation, 'Preparing server files', { phase: 'prepare', recipeStatus: 'attempted', blocker: null, checks: [],
+            checkpoint: { key: 'startup-queue', label: 'Admitted to startup' },
+            event: { kind: 'recipe', message: 'Promoted files selected for this creation attempt.', details: { capturedAt: recipe.capturedAt } } });
+    })();
     const configFile = recipe.configFile;
-    deploymentStage(operation, 'Preparing server files');
     const hostDir = serverDir(name);
 
     // Step 1: filesystem. mkdir mode is masked by umask, so chmod explicitly
@@ -398,13 +418,18 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
     }
 
     // Step 2: Komodo CreateDeployment.
-    deploymentStage(operation, 'Creating Komodo deployment');
+    deploymentStage(operation, 'Creating Komodo deployment', { phase: 'create-deployment',
+        checkpoint: { key: 'prepare', label: 'Server files prepared' },
+        command: { state: 'attempted', message: 'Komodo deployment creation attempted.' } });
     const spec = buildSpec({ game, name, port, hostDir, plutoKey, recipe });
-    const createRes = await komodoCreateDeployment(spec);
+    const createRes = await observeCommand(operation, () => komodoCreateDeployment(spec));
     if (!createRes.ok) {
+        recordProgress(operation, { command: { state: createRes.kind === 'unreachable' || (createRes.httpStatus !== undefined && createRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: createRes.message } });
         await rm(hostDir, { recursive: true, force: true }).catch(() => {});
         return { ok: false, kind: 'komodo', message: `CreateDeployment: ${createRes.message}` };
     }
+    recordProgress(operation, { command: { state: 'accepted', message: 'Komodo created the deployment.' },
+        checkpoint: { key: 'remote-deployment-created', phase: 'create-deployment', label: 'Komodo deployment created' } });
 
     // Step 3: DB row — insert *before* execute/Deploy so the sweep sees the
     // server immediately even if Deploy hangs.
@@ -446,23 +471,31 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
     void applyDeploymentTags(name, game, mapEntry.canonical);
 
     // Step 5: execute/Deploy — actually start the container.
-    deploymentStage(operation, 'Starting container');
-    db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operation);
-    const deployRes = await komodoDeploy(name);
+    db().transaction(() => {
+        deploymentStage(operation, 'Starting container', { phase: 'container-action',
+            checkpoint: { key: 'create-deployment', label: 'Deployment created and server registered' },
+            command: { state: 'attempted', message: 'Container startup submission attempted.' } });
+        db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operation);
+    })();
+    const deployRes = await observeCommand(operation, () => komodoDeploy(name));
     if (!deployRes.ok) {
         if (deployRes.kind === 'unreachable' || (deployRes.httpStatus !== undefined && deployRes.httpStatus >= 500)) {
             // A timeout or server error may arrive after the remote boot began.
             // Preserve files, identity, and capacity until readiness is known.
             recordDeploymentEvent(operation, 'Startup response uncertain; verifying the existing server: ' + deployRes.message);
+            recordProgress(operation, { command: { state: 'uncertain', message: 'Startup may have begun: ' + deployRes.message } });
             return { ok: true, name, port, deploymentId: operation };
         }
         // Container never started — undo everything so retrying is clean.
+        recordProgress(operation, { command: { state: 'rejected', message: 'Startup request rejected: ' + deployRes.message } });
         await komodoDeleteDeployment(name).catch(() => {});
         db().prepare('DELETE FROM servers WHERE name = ?').run(name);
         await rm(hostDir, { recursive: true, force: true }).catch(() => {});
         return { ok: false, kind: 'komodo', message: `Deploy: ${deployRes.message}` };
     }
 
+    recordProgress(operation, { command: { state: 'accepted', message: 'Komodo accepted startup; readiness still requires verification.' },
+        checkpoint: { key: 'container-action', label: 'Container startup accepted' } });
     return { ok: true, name, port, deploymentId: operation };
 }
 
@@ -515,7 +548,8 @@ export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerRe
     const operation = db().transaction(() => {
         const id = beginDeployment(name, 'destroy', triggeredBy);
         db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
-        deploymentStage(id, 'Waiting to verify the server is empty');
+        deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
+            blocker: { code: 'eligibility', message: 'Waiting for safe removal eligibility checks.', automatic: true } });
         return id;
     })();
     return { ok: true, warnings: [], deploymentId: operation };
@@ -528,6 +562,13 @@ export function processDestructions(): Promise<void> {
     return destructionTick;
 }
 
+function betaRemovalEvidence(server: string, message: string): ProvisioningBlocker {
+    const beta = db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
+        WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') ORDER BY b.id LIMIT 1`)
+        .get(server) as { id: number } | undefined;
+    return { code: 'active-beta', message, automatic: true, href: beta ? `/betas/${beta.id}` : null, label: 'Blocking beta' };
+}
+
 async function runDestructions(): Promise<void> {
     const queued = db().prepare(`SELECT id, server, stage, server_deleted_at FROM server_deployments WHERE action = 'destroy' AND state = 'waiting'
         AND (server_deleted_at IS NULL OR stage = ?) ORDER BY started_at`).all(REMOVING_SERVER) as { id: string; server: string; stage: string; server_deleted_at: string | null }[];
@@ -553,37 +594,53 @@ async function runDestructions(): Promise<void> {
             }
             if (!getServer(operation.server)) {
                 if (hasActiveCreation(operation.server)) {
-                    deploymentStage(operation.id, 'Waiting for server creation to finish');
+                    const creatingOperation = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL
+                        AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health') LIMIT 1`).get(operation.server) as { id: string } | undefined;
+                    deploymentStage(operation.id, 'Waiting for server creation to finish', { phase: 'eligibility', checks: [],
+                        blocker: { code: 'creation-pending', message: 'Waiting for server creation to finish', automatic: true,
+                            href: creatingOperation ? `/provisioning/${encodeURIComponent(creatingOperation.id)}` : null, label: 'Creation operation' } });
                     continue;
                 }
                 finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' });
                 continue;
             }
             const blocker = activeBetaRemovalBlocker(operation.server);
-            if (blocker) { deploymentStage(operation.id, blocker); continue; }
+            if (blocker) { deploymentStage(operation.id, blocker, { phase: 'eligibility', checks: [], blocker: betaRemovalEvidence(operation.server, blocker) }); continue; }
             // A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
-            let players = await probePlayerCount(operation.server);
+            const evidence = await probePlayerEvidence(operation.server);
+            let players = evidence.players;
+            let occupancy: NonNullable<ProgressUpdate['checks']>[number] = { key: 'occupancy', label: 'Player occupancy',
+                result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting', source: evidence.source,
+                checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players };
             if (players === null) {
                 const health = await komodoInspectHealthy(operation.server);
-                if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
+                if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) {
+                    players = 0;
+                    occupancy = { key: 'stopped', label: 'Stopped container permits removal', result: 'pass', source: 'komodo', actual: health.state ?? null };
+                }
             }
             if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
             // Beta ownership may have changed while the network probes were in flight.
             const latestBlocker = activeBetaRemovalBlocker(operation.server);
-            if (latestBlocker) { deploymentStage(operation.id, latestBlocker); continue; }
+            if (latestBlocker) { deploymentStage(operation.id, latestBlocker, { phase: 'eligibility', checks: [occupancy], blocker: betaRemovalEvidence(operation.server, latestBlocker) }); continue; }
             if (players !== 0) {
-                deploymentStage(operation.id, players === null
+                const message = players === null
                     ? 'Waiting for player count: server could not be reached'
-                    : 'Waiting for server to empty: ' + players + ' player(s) online');
+                    : 'Waiting for server to empty: ' + players + ' player(s) online';
+                deploymentStage(operation.id, message, { phase: 'eligibility', checks: [occupancy],
+                    blocker: { code: players === null ? 'players-unknown' : 'players-present', message, automatic: true } });
                 continue;
             }
             if (!claimRemoval(operation.id)) continue;
-            deploymentStage(operation.id, 'Server is empty (0 players verified)');
+            deploymentStage(operation.id, 'Server is empty (0 players verified)', { phase: 'eligibility', checks: [occupancy], blocker: null,
+                checkpoint: { key: 'eligibility', label: 'Removal eligibility verified' } });
             const result = await destroyServerUnlocked(operation.server, operation.id);
             finishDeployment(operation.id, result);
         } catch (err) {
             const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string };
-            if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message);
+            if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message, {
+                phase: 'eligibility', checks: [], blocker: { code: 'players-unknown', message: (err as Error).message, automatic: true }
+            });
             else if (row.state !== 'cancelled') finishDeployment(operation.id, { ok: false, message: (err as Error).message });
         } finally { destroying.delete(operation.server); }
     }
@@ -601,11 +658,15 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
 
     // 1: stop + remove the container. execute/DestroyDeployment queues the
     // teardown; the response comes back before the container is actually gone.
-    deploymentStage(operation, 'Stopping and removing container');
-    const destroyRes = await komodoDestroyDeployment(name);
+    deploymentStage(operation, 'Stopping and removing container', { phase: 'remove-container', blocker: null, checks: [],
+        checkpoint: { key: 'eligibility', label: 'Removal eligibility verified' },
+        command: { state: 'attempted', message: 'Container removal submission attempted.' } });
+    const destroyRes = await observeCommand(operation, () => komodoDestroyDeployment(name));
     if (!destroyRes.ok) {
+        recordProgress(operation, { command: { state: destroyRes.kind === 'unreachable' || (destroyRes.httpStatus !== undefined && destroyRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: destroyRes.message } });
         return { ok: false, kind: 'komodo', message: 'DestroyDeployment failed: ' + destroyRes.message, warnings };
     }
+    recordProgress(operation, { command: { state: 'accepted', message: 'Komodo accepted teardown; removal is not yet confirmed.' } });
 
     // 2: wait for the container to actually disappear. Komodo refuses to
     // DeleteDeployment while a container still exists for it, so poll the
@@ -619,12 +680,19 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
     }
 
     // 3: remove the deployment entity itself.
-    deploymentStage(operation, 'Deleting Komodo deployment');
-    const deleteRes = await komodoDeleteDeployment(name);
+    deploymentStage(operation, 'Deleting Komodo deployment', { phase: 'delete-deployment',
+        command: { state: 'attempted', message: 'Komodo deployment deletion attempted.' } });
+    const deleteRes = await observeCommand(operation, () => komodoDeleteDeployment(name));
     if (!deleteRes.ok) {
+        recordProgress(operation, { command: { state: deleteRes.kind === 'unreachable' || (deleteRes.httpStatus !== undefined && deleteRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: deleteRes.message } });
         return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings };
     }
 
+    deploymentStage(operation, REMOVING_SERVER, { phase: 'cleanup', blocker: null, checks: [],
+        command: { state: 'accepted', message: 'Komodo confirmed deployment deletion.' },
+        checkpoints: [{ key: 'remove-container', label: 'Container removal confirmed by deployment deletion' },
+            { key: 'delete-deployment', label: 'Komodo deployment deletion confirmed' }] });
+
     return cleanupDestroyedServer(name, operation);
 }
 
@@ -635,12 +703,14 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
     const warnings: string[] = [];
     // Persist the checkpoint before local work so failed cleanup can resume without RCON.
     // Legacy operations already recorded this same stage after Komodo deletion.
-    deploymentStage(operation, REMOVING_SERVER);
+    deploymentStage(operation, REMOVING_SERVER, { phase: 'cleanup', blocker: null, checks: [],
+        checkpoint: { key: 'delete-deployment', label: 'Komodo deployment deletion confirmed' } });
     // Host directory. Fall back to derived path if host_dir wasn't set on
     // legacy rows.
     const hostDir = row?.host_dir ?? serverDir(name);
     try {
         await rm(hostDir, { recursive: true, force: true });
+        recordProgress(operation, { checkpoint: { key: 'files-removed', label: 'Instance files removed', phase: 'cleanup' } });
     } catch (err) {
         warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
     }
@@ -648,7 +718,14 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
     // The registry trigger preserves history and marks its old server identity.
     // Live inventory, rollback snapshots, and restart slots still cascade away.
     try {
-        db().prepare('DELETE FROM servers WHERE name = ?').run(name);
+        db().transaction(() => {
+            const recovery = db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ? AND healthy_at IS NULL').get(name) as { operation_id: string | null } | undefined;
+            db().prepare('DELETE FROM servers WHERE name = ?').run(name);
+            if (recovery?.operation_id) recordProgress(recovery.operation_id, { recovery: { state: 'released', operationId: recovery.operation_id,
+                message: 'Recovery capacity released when the old instance registry was removed.' } });
+            recordProgress(operation, { checkpoint: { key: 'registry-removed', label: 'Server registration removed', phase: 'cleanup' } });
+            if (!warnings.length) recordProgress(operation, { checkpoint: { key: 'cleanup', label: 'Local files and registry cleanup completed' } });
+        })();
     } catch (err) {
         return {
             ok: false,
@@ -684,8 +761,11 @@ async function provisionCreation(id: string): Promise<CreateServerResult> {
             { ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
         if (!result.ok) finishDeployment(id, result);
         else {
-            db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
-            deploymentStage(id, 'Waiting for Komodo health');
+            db().transaction(() => {
+                db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
+                deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', checks: [],
+                    blocker: { code: 'container-readiness', message: 'Waiting for container readiness.', automatic: true } });
+            })();
         }
         return result;
     } catch (err) {
@@ -693,9 +773,13 @@ async function provisionCreation(id: string): Promise<CreateServerResult> {
         if (server?.game_port === job.port) {
             // A lost response may have happened after Komodo started the server.
             // Keep its slot and observe readiness instead of starting another boot.
-            db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
-            recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message);
-            deploymentStage(id, 'Verifying existing server after interrupted startup');
+            db().transaction(() => {
+                db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
+                recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message);
+                deploymentStage(id, 'Verifying existing server after interrupted startup', { phase: 'verify-container', checks: [],
+                    command: { state: 'uncertain', message: 'Startup response interrupted: ' + (err as Error).message },
+                    blocker: { code: 'container-readiness', message: 'Observing the existing instance after an uncertain startup.', automatic: true } });
+            })();
         } else finishDeployment(id, { ok: false, message: (err as Error).message });
         throw err;
     } finally { creating = false; }
@@ -722,19 +806,32 @@ export function creationQueue() {
     return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
 }
 
-function creationBlocker(id: string): string | null {
-    const dependency = db().prepare(`SELECT d.server, d.state FROM server_creation_dependencies dep
+function creationBlocker(id: string): ProvisioningBlocker | null {
+    const dependency = db().prepare(`SELECT d.id, d.server, d.state FROM server_creation_dependencies dep
         JOIN server_deployments d ON d.id = dep.removal_id
-        WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { server: string; state: string } | undefined;
-    if (dependency) return `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`;
+        WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { id: string; server: string; state: string } | undefined;
+    if (dependency) return { code: 'removal-dependency', message: `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`,
+        automatic: true, href: `/provisioning/${encodeURIComponent(dependency.id)}`, label: 'Required removal' };
     const occupied = db().prepare(`SELECT s.name FROM servers s JOIN server_creation_jobs j ON s.game_port = j.port
         WHERE j.deployment_id = ? LIMIT 1`).get(id) as { name: string } | undefined;
-    return occupied ? `Waiting for port to be released by ${occupied.name}` : null;
+    return occupied ? { code: 'port-occupied', message: `Waiting for port to be released by ${occupied.name}`, automatic: true,
+        href: `/servers/${encodeURIComponent(occupied.name)}`, label: 'Server using this port' } : null;
+}
+
+function stillCheckingCreation(id: string): boolean {
+    return !!db().prepare(`SELECT 1 FROM server_deployments d JOIN server_creation_jobs j ON j.deployment_id = d.id
+        JOIN servers s ON s.name = j.server AND s.game_port = j.port
+        WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state = 'awaiting-health'`).get(id);
 }
 
 async function runCreations(): Promise<CreationOutcomes> {
     const outcomes: CreationOutcomes = new Map();
-    if (creating || destroying.size > 0) return outcomes;
+    if (creating || destroying.size > 0) {
+        for (const job of creationQueue().queued) recordProgress(job.id, { phase: 'startup-queue', blocker: {
+            code: 'startup-worker-busy', message: 'Waiting for in-flight instance creation or removal to finish.', automatic: true
+        } });
+        return outcomes;
+    }
     // Restart recovery is observation-only: never repeat external creation or
     // deployment. A known interrupted startup must retain its readiness gate.
     const interrupted = db().prepare("SELECT id FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
@@ -752,7 +849,8 @@ async function runCreations(): Promise<CreationOutcomes> {
                 AND newer.rowid > d.rowid AND newer.action = 'create')`).get(id);
         if (removing) {
             db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
-            deploymentStage(id, 'Waiting for requested server removal after interrupted startup');
+            deploymentStage(id, 'Waiting for requested server removal after interrupted startup', { checks: [],
+                blocker: { code: 'removal-pending', message: 'Waiting for the requested removal; the interrupted startup still occupies capacity.', automatic: true } });
         }
     }
 
@@ -766,21 +864,51 @@ async function runCreations(): Promise<CreationOutcomes> {
             finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' });
             continue;
         }
-        if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
+        const removal = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server) as { id: string } | undefined;
+        if (removal) {
+            recordProgress(id, { checks: [], blocker: { code: 'removal-pending', message: 'Creation verification is held while server removal is pending.', automatic: true,
+                href: `/provisioning/${encodeURIComponent(removal.id)}`, label: 'Removal operation' } });
+            continue;
+        }
+        const checks: NonNullable<ProgressUpdate['checks']> = [];
         try {
             const komodo = await komodoInspectHealthy(job.server);
-            if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
+            if (!stillCheckingCreation(id)) continue;
+            checks.push({ key: 'container', label: 'Container readiness', source: 'komodo', result: komodo.ok ? 'pass' : 'waiting',
+                checkedAt: new Date().toISOString(), actual: komodo.state ?? null, message: komodo.ok ? null : komodo.reason });
+            if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason, { phase: 'verify-container', checks,
+                blocker: { code: 'container-readiness', message: komodo.reason, automatic: true } }); continue; }
             const status = await probeServerStatus(job.server, job.submitted_at);
-            if (!status.ok) { deploymentStage(id, 'Waiting for RCON or fresh telemetry health: ' + status.reason); continue; }
+            if (!stillCheckingCreation(id)) continue;
+            checks.push({ key: 'game', label: 'Game status', source: status.ok ? status.source : null, result: status.ok ? 'pass' : 'waiting',
+                checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null, message: status.ok ? null : status.reason });
+            if (!status.ok) { deploymentStage(id, 'Waiting for RCON or fresh telemetry health: ' + status.reason, { phase: 'verify-game', checks,
+                blocker: { code: 'game-readiness', message: status.reason, automatic: true } }); continue; }
             const actualMap = status.map;
+            checks.push({ key: 'map', label: 'Selected engine map', source: status.source, result: actualMap === job.expected_map ? 'pass' : 'waiting',
+                checkedAt: status.observedAt, receivedAt: status.receivedAt, expected: job.expected_map, actual: actualMap });
             if (actualMap !== job.expected_map) {
-                deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', ' + (status.source === 'rcon' ? 'RCON' : 'telemetry') + ' reports ' + (actualMap ?? 'no map'));
+                const message = 'Waiting for selected map: expected ' + job.expected_map + ', ' + (status.source === 'rcon' ? 'RCON' : 'telemetry') + ' reports ' + (actualMap ?? 'no map');
+                deploymentStage(id, message, { phase: 'verify-map', checks, blocker: { code: 'map-mismatch', message, automatic: true,
+                    href: `/servers/${encodeURIComponent(job.server)}?tab=config`, label: 'Map configuration' } });
                 continue;
             }
-            deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap);
-            finishDeployment(id, { ok: true });
+            db().transaction(() => {
+                deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap, {
+                    phase: 'verify-map', checks, blocker: null, checkpoints: [
+                        { key: 'container-action', label: 'Container startup verified' },
+                        { key: 'verify-container', label: 'Container readiness verified' },
+                        { key: 'verify-game', label: 'Game status verified' },
+                        { key: 'verify-map', label: 'Selected engine map verified' }
+                    ]
+                });
+                finishDeployment(id, { ok: true });
+            })();
         } catch (err) {
-            deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message);
+            if (!stillCheckingCreation(id)) continue;
+            checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', source: null, result: 'unknown', message: (err as Error).message });
+            deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message, { phase: checks.length > 1 ? 'verify-game' : 'verify-container', checks,
+                blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true } });
         }
     }
 
@@ -790,7 +918,9 @@ async function runCreations(): Promise<CreationOutcomes> {
     const available = Math.max(0, fleetSettings().createConcurrency - queue.active.length);
     const eligible = queue.queued.filter(({ id }) => {
         const blocker = creationBlocker(id);
-        if (blocker) deploymentStage(id, blocker);
+        if (blocker) deploymentStage(id, blocker.message, { phase: 'startup-queue', blocker });
+        else recordProgress(id, { phase: 'startup-queue', blocker: { code: available ? 'startup-queue' : 'startup-capacity',
+            message: available ? 'Eligible for startup admission in request order.' : 'Waiting for startup capacity to be released.', automatic: true } });
         return !blocker;
     });
     for (const { id } of eligible.slice(0, available)) {
@@ -801,7 +931,7 @@ async function runCreations(): Promise<CreationOutcomes> {
         if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
         const blocker = creationBlocker(id);
         if (blocker) {
-            deploymentStage(id, blocker);
+            deploymentStage(id, blocker.message, { phase: 'startup-queue', blocker });
             continue;
         }
         try { outcomes.set(id, await provisionCreation(id)); }

src/lib/server/provisioning/progress.ts

@@ -0,0 +1,120 @@
+import { db } from '../db/index.js';
+import { provisioningPhases, type ProvisioningPhase, type ProvisioningProgress, type ProvisioningCheck,
+    type ProvisioningCheckpoint, type ProvisioningBlocker, type RecoveryObservation } from '../../provisioning-detail.js';
+
+export type CheckInput = Omit<ProvisioningCheck, 'checkedAt' | 'receivedAt' | 'expected' | 'actual' | 'message'> &
+    Partial<Pick<ProvisioningCheck, 'checkedAt' | 'receivedAt' | 'expected' | 'actual' | 'message'>>;
+export type CheckpointInput = Omit<ProvisioningCheckpoint, 'at' | 'phase'> & { at?: string; phase?: ProvisioningPhase | null };
+export interface ProgressUpdate {
+    phase?: ProvisioningPhase | null;
+    blocker?: ProvisioningBlocker | null;
+    /** A complete evaluation; omitted checks are no longer current evidence. */
+    checks?: CheckInput[];
+    checkpoint?: CheckpointInput;
+    checkpoints?: CheckpointInput[];
+    command?: (Omit<NonNullable<ProvisioningProgress['command']>, 'at'> & { at?: string }) | null;
+    recovery?: (Omit<RecoveryObservation, 'at'> & { at?: string }) | null;
+    recipeStatus?: ProvisioningProgress['recipeStatus'];
+    observedAt?: string | null;
+    event?: { kind: string; message: string; details?: Record<string, unknown> };
+}
+
+const object = (value: unknown): value is Record<string, unknown> => !!value && typeof value === 'object' && !Array.isArray(value);
+const nullableString = (value: unknown) => value === null || typeof value === 'string';
+const phase = (value: unknown): value is ProvisioningPhase => provisioningPhases.includes(value as ProvisioningPhase);
+const scalar = (value: unknown) => nullableString(value) || typeof value === 'number' && Number.isFinite(value);
+
+/** Invalid/old snapshots never manufacture evidence. Raw legacy stages remain available. */
+export function readProgress(row: { observation_json?: string | null }): ProvisioningProgress | null {
+    if (!row.observation_json) return null;
+    try {
+        const v: unknown = JSON.parse(row.observation_json);
+        if (!object(v) || v.version !== 1 || !(v.phase === null || phase(v.phase)) || !nullableString(v.phaseStartedAt)
+            || !nullableString(v.observedAt) || !Array.isArray(v.checks) || !Array.isArray(v.checkpoints)) return null;
+        if (v.blocker !== null && (!object(v.blocker) || typeof v.blocker.code !== 'string' || typeof v.blocker.message !== 'string'
+            || typeof v.blocker.automatic !== 'boolean' || (v.blocker.href !== undefined && !nullableString(v.blocker.href))
+            || (v.blocker.label !== undefined && !nullableString(v.blocker.label)))) return null;
+        if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'occupancy', 'stopped'].includes(String(c.key))
+            && typeof c.label === 'string' && ['pass', 'waiting', 'fail', 'unknown'].includes(String(c.result))
+            && (c.source === null || ['komodo', 'rcon', 'telemetry', 'combined'].includes(String(c.source)))
+            && typeof c.checkedAt === 'string' && nullableString(c.receivedAt) && scalar(c.expected) && scalar(c.actual) && nullableString(c.message))) return null;
+        if (v.checkpoints.length > 100 || !v.checkpoints.every(c => object(c) && typeof c.key === 'string' && typeof c.label === 'string'
+            && typeof c.at === 'string' && (c.phase === null || phase(c.phase)))) return null;
+        if (v.command !== null && (!object(v.command) || !['attempted', 'accepted', 'rejected', 'uncertain'].includes(String(v.command.state))
+            || typeof v.command.at !== 'string' || typeof v.command.message !== 'string')) return null;
+        if (v.recovery !== null && (!object(v.recovery) || !['held', 'released', 'transferred', 'unknown'].includes(String(v.recovery.state))
+            || typeof v.recovery.at !== 'string' || typeof v.recovery.message !== 'string'
+            || (v.recovery.operationId !== undefined && !nullableString(v.recovery.operationId)))) return null;
+        if (v.recipeStatus !== null && v.recipeStatus !== 'queued' && v.recipeStatus !== 'attempted') return null;
+        return v as unknown as ProvisioningProgress;
+    } catch { return null; }
+}
+
+/**
+ * Records facts at the existing execution boundary, never performs a probe or command.
+ * Can be nested inside the caller's transaction with its state/legacy-stage writes.
+ */
+export function recordProgress(id: string, update: ProgressUpdate): void {
+    const handle = db();
+    handle.transaction(() => {
+        const row = handle.prepare('SELECT observation_json FROM server_deployments WHERE id = ?').get(id) as
+            { observation_json: string | null } | undefined;
+        if (!row) return;
+        const now = new Date().toISOString();
+        const previous = readProgress(row);
+        const next: ProvisioningProgress = previous ? { ...previous, checkpoints: [...previous.checkpoints] } : {
+            version: 1, phase: null, phaseStartedAt: null, observedAt: null, blocker: null, checks: [], checkpoints: [],
+            command: null, recovery: null, recipeStatus: null
+        };
+        const events: { kind: string; message: string; details?: Record<string, unknown> }[] = [];
+        if (update.phase !== undefined && update.phase !== next.phase) {
+            next.phase = update.phase;
+            next.phaseStartedAt = update.phase === null ? null : now;
+            next.blocker = null;
+        }
+        next.observedAt = update.observedAt === undefined ? now : update.observedAt;
+        if (update.blocker !== undefined) {
+            const old = next.blocker;
+            next.blocker = update.blocker;
+            // Counts/error wording can change on every poll without a new kind of hold.
+            if (update.blocker && (old?.code !== update.blocker.code || old?.href !== update.blocker.href)) {
+                events.push({ kind: 'blocker', message: update.blocker.message,
+                    details: { code: update.blocker.code, href: update.blocker.href ?? null } });
+            }
+        }
+        if (update.checks !== undefined) {
+            next.checks = update.checks.map(check => ({ ...check, checkedAt: check.checkedAt ?? now,
+                receivedAt: check.receivedAt ?? null, expected: check.expected ?? null, actual: check.actual ?? null,
+                message: check.message ?? null }));
+        }
+        for (const input of [...(update.checkpoints ?? []), ...(update.checkpoint ? [update.checkpoint] : [])]) {
+            if (next.checkpoints.some(c => c.key === input.key)) continue;
+            const point: ProvisioningCheckpoint = { ...input, at: input.at ?? now, phase: input.phase ?? (phase(input.key) ? input.key : next.phase) };
+            next.checkpoints.push(point);
+            events.push({ kind: 'checkpoint', message: point.label, details: { key: point.key, at: point.at } });
+        }
+        if (update.command !== undefined) {
+            if (!update.command) next.command = null;
+            else if (next.command?.state !== update.command.state || next.command?.message !== update.command.message) {
+                next.command = { ...update.command, at: update.command.at ?? now };
+                events.push({ kind: 'command', message: next.command.message, details: { state: next.command.state, at: next.command.at } });
+            }
+        }
+        if (update.recovery !== undefined) {
+            if (!update.recovery) next.recovery = null;
+            else {
+                const changed = next.recovery?.state !== update.recovery.state || next.recovery?.operationId !== update.recovery.operationId;
+                next.recovery = { ...update.recovery, at: changed ? update.recovery.at ?? now : next.recovery!.at };
+                if (changed) events.push({ kind: 'recovery', message: next.recovery.message,
+                    details: { state: next.recovery.state, at: next.recovery.at, operationId: next.recovery.operationId ?? null } });
+            }
+        }
+        if (update.recipeStatus !== undefined) next.recipeStatus = update.recipeStatus;
+        if (update.event) events.push(update.event);
+        handle.prepare(`UPDATE server_deployments SET phase_key = ?, phase_started_at = ?, observed_at = ?, observation_json = ? WHERE id = ?`)
+            .run(next.phase, next.phaseStartedAt, next.observedAt, JSON.stringify(next), id);
+        const insert = handle.prepare(`INSERT INTO server_deployment_events (deployment_id, created_at, message, kind, phase_key, details_json)
+            VALUES (?, ?, ?, ?, ?, ?)`);
+        for (const event of events) insert.run(id, now, event.message, event.kind, next.phase, event.details ? JSON.stringify(event.details) : null);
+    })();
+}

src/lib/server/provisioning/requests.ts

@@ -6,6 +6,7 @@ import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEve
 import { queueRebuildServer } from './instances.js';
 import { restartRequestInFlight } from './restarts.js';
 import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
+import { recordProgress } from './progress.js';
 
 export interface LifecycleOperation {
     id: string; server: string; action: string; state: string; stage: string;
@@ -67,7 +68,8 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
         if (!id) {
             id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
             db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
-            deploymentStage(id, 'Waiting to verify the server is empty');
+            deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
+                blocker: { code: 'eligibility', message: 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } });
             if (row.restarted_at) adoptSubmittedRestart(id, server, row.restarted_at);
         }
         db().prepare('INSERT OR IGNORE INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, ?, ?, ?)')
@@ -79,12 +81,17 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
 
 /** Upgrade observation only: existing restart intent is never resubmitted. */
 function adoptSubmittedRestart(id: string, server: string, submittedAt: string): void {
-    const slot = db().prepare('SELECT operation_id, requested_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as
-        { operation_id: string | null; requested_at: string; healthy_at: string | null; players: number | null } | undefined;
+    const slot = db().prepare('SELECT operation_id, requested_at, checked_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as
+        { operation_id: string | null; requested_at: string; checked_at: string | null; healthy_at: string | null; players: number | null } | undefined;
     db().prepare("UPDATE server_deployments SET state = 'awaiting-health', submitted_at = ? WHERE id = ?").run(submittedAt, id);
+    recordProgress(id, { command: { state: 'attempted', at: submittedAt, message: 'Adopted a previously recorded submission; no command is replayed.' } });
     if (slot && ((slot.operation_id && slot.requested_at >= submittedAt) || (slot.requested_at < submittedAt && !slot.healthy_at))) {
+        const finishedAt = new Date().toISOString();
+        const error = 'Existing restart cannot be correlated safely. Inspect the server before retrying.';
         db().prepare("UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE id = ?")
-            .run(new Date().toISOString(), 'Existing restart cannot be correlated safely. Inspect the server before retrying.', id);
+            .run(finishedAt, error, id);
+        recordProgress(id, { recovery: { state: 'unknown', message: 'Existing restart recovery could not be correlated with this operation.', operationId: id },
+            observedAt: null, event: { kind: 'outcome', message: error, details: { state: 'interrupted', finishedAt, error } } });
         return;
     }
     if (slot && slot.requested_at >= submittedAt) {
@@ -96,16 +103,24 @@ function adoptSubmittedRestart(id: string, server: string, submittedAt: string):
     }
     if (slot?.healthy_at && slot.requested_at >= submittedAt) {
         db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ?, players = ? WHERE id = ?").run(slot.healthy_at, slot.players, id);
+        recordProgress(id, { observedAt: slot.healthy_at, event: { kind: 'outcome', message: 'Adopted recorded recovery completion.',
+            details: { state: 'done', finishedAt: slot.healthy_at, error: null, warnings: [] } } });
     }
-    deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again');
+    deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again', {
+        phase: 'verify-container', checks: [], blocker: null, observedAt: slot?.checked_at ?? null,
+        recovery: { state: slot?.healthy_at && slot.requested_at >= submittedAt ? 'released' : 'held',
+            at: slot?.healthy_at && slot.requested_at >= submittedAt ? slot.healthy_at : undefined,
+            message: slot?.healthy_at && slot.requested_at >= submittedAt ? 'Adopted recorded recovery completion.' : 'Observing the previously submitted restart without replaying it.', operationId: id }
+    });
 }
 
 export function cancelProvisioning(id: string, reason = 'Cancelled by requester'): boolean {
     return db().transaction(() => {
+        const finishedAt = new Date().toISOString();
         const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL
             WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL`)
-            .run(new Date().toISOString(), reason, id);
-        if (result.changes) recordDeploymentEvent(id, reason);
+            .run(finishedAt, reason, id);
+        if (result.changes) recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome', message: reason, details: { state: 'cancelled', finishedAt, error: null, warnings: [] } } });
         return result.changes > 0;
     })();
 }
@@ -143,7 +158,8 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
         const id = beginDeployment(server, action);
         db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
         db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
-        deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty');
+        deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
+            blocker: { code: 'eligibility', message: action === 'start' ? 'Waiting for start eligibility and recovery capacity.' : 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } });
         return { ok: true as const, deploymentId: id };
     })();
 }
@@ -177,7 +193,8 @@ function queueWatchdogAction(server: string, action: 'restart' | 'rebuild', inci
     } else {
         id = beginDeployment(server, action, `watchdog:${incidentId}`);
         db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
-        deploymentStage(id, 'Waiting to verify the server is empty');
+        deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
+            blocker: { code: 'eligibility', message: 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } });
     }
     db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'watchdog', ?, ?)").run(id, incidentId, server);
     return { ok: true, deploymentId: id };
@@ -209,8 +226,14 @@ export function requestWatchdogLifecycle(server: string, action: 'restart' | 're
             WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
             AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined;
         if (retained && !restartRequestInFlight(server)) {
-            db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
+            const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
                 .run(next.deploymentId, server, retained.operation_id);
+            if (transferred.changes) {
+                recordProgress(retained.operation_id, { recovery: { state: 'transferred', operationId: next.deploymentId,
+                    message: 'Recovery capacity transferred to the explicit new incident attempt.' } });
+                recordProgress(next.deploymentId, { recovery: { state: 'held', operationId: next.deploymentId,
+                    message: 'Recovery capacity retained from the prior incident attempt.' } });
+            }
             recordDeploymentEvent(retained.operation_id, 'Recovery capacity handed to explicit Watchdog retry ' + next.deploymentId);
             recordDeploymentEvent(next.deploymentId, 'Retained recovery capacity from prior incident attempt ' + retained.operation_id);
         }
@@ -249,8 +272,16 @@ export function escalateWatchdogRestart(incidentId: string, operationId: string,
         const reason = 'Watchdog restart recovery timed out; handed recovery to rebuild ' + replacement.deploymentId;
         if (operation.state === 'awaiting-health') finishDeployment(operationId, { ok: false, message: reason });
         else recordDeploymentEvent(operationId, reason);
-        if (slot) db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
-            .run(replacement.deploymentId, reason, operation.server, operationId);
+        if (slot) {
+            const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
+                .run(replacement.deploymentId, reason, operation.server, operationId);
+            if (transferred.changes) {
+                recordProgress(operationId, { recovery: { state: 'transferred', operationId: replacement.deploymentId,
+                    message: 'Recovery capacity transferred to the rebuild operation.' } });
+                recordProgress(replacement.deploymentId, { recovery: { state: 'held', operationId: replacement.deploymentId,
+                    message: 'Recovery capacity retained from the previous restart.' } });
+            }
+        }
         recordDeploymentEvent(replacement.deploymentId, 'Recovery handed off from restart ' + operationId + '; waiting for a fresh empty or stopped server');
         return replacement;
     })();

src/lib/server/provisioning/restarts.ts

@@ -5,6 +5,8 @@ import { probeServerStatus } from '../fleet/status-evidence.js';
 import { deploymentStage, finishDeployment } from './history.js';
 import { applyStagedFilesForOperation } from '../fleet/staged-files.js';
 import type { RuntimeAction } from '../../provisioning.js';
+import type { ProvisioningBlocker } from '../../provisioning-detail.js';
+import { recordProgress, type ProgressUpdate } from './progress.js';
 
 // Komodo queues restart work. Don't mistake the still-running old container
 // for a completed restart when an API kick immediately follows the request.
@@ -38,21 +40,28 @@ export function restartSettling(server: string): boolean {
 }
 
 export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null {
-    if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
+    return rolloutBlocker(server, operationId, action)?.message ?? null;
+}
+
+export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null {
+    const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
         AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
-            OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) {
-        return 'Waiting for the current provisioning operation';
+            OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined;
+    if (conflict) {
+        return { code: 'operation-conflict', message: 'Waiting for the current provisioning operation', automatic: true,
+            href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' };
     }
     const slot = slotFor(server);
     const ownHandoff = slot?.operation_id === operationId && !!db().prepare(`SELECT 1 FROM server_deployments d
         WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS (
             SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId);
     if (slot?.healthy_at === null && !ownHandoff) {
-        return `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`;
+        return { code: 'previous-recovery', message: `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`, automatic: true,
+            href: slot.operation_id ? `/provisioning/${encodeURIComponent(slot.operation_id)}` : null, label: 'Previous operation' };
     }
     const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
     return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= fleetSettings().restartConcurrency
-        ? `Queued to ${action}; waiting for another server to recover` : null;
+        ? { code: 'restart-capacity', message: `Queued to ${action}; waiting for another server to recover`, automatic: true } : null;
 }
 
 export type RolloutRestartResult =
@@ -71,6 +80,7 @@ export async function restartInRollout(server: string, options: { operationId: s
             ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
                 checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
             .run(server, new Date().toISOString(), operationId);
+        recordProgress(operationId, { blocker: null, recovery: { state: 'held', message: 'Restart recovery capacity reserved.', operationId } });
         return null;
     }).immediate();
     if (waitReason) return { kind: 'queued', message: waitReason };
@@ -81,12 +91,20 @@ export async function restartInRollout(server: string, options: { operationId: s
         applyStagedFilesForOperation(operationId, server);
     } catch (err) {
         const message = `File activation failed: ${(err as Error).message}`;
-        db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
-        finishDeployment(operationId, { ok: false, message });
+        db().transaction(() => {
+            db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
+            recordProgress(operationId, { recovery: { state: 'released', message: 'Capacity released before any remote command: file activation failed.', operationId } });
+            finishDeployment(operationId, { ok: false, message });
+        })();
         return { kind: 'failed', message };
     }
-    db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
-    deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container');
+    db().transaction(() => {
+        db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
+        deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container', {
+            phase: 'container-action', blocker: null, checks: [], checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' },
+            command: { state: 'attempted', message: `${options.action === 'start' ? 'Start' : 'Restart'} submission attempted.` }
+        });
+    })();
     requestsInFlight.set(server, operationId);
     db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
         game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
@@ -94,24 +112,38 @@ export async function restartInRollout(server: string, options: { operationId: s
         const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
         if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
         if (result.ok) {
-            db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId);
-            deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)');
+            db().transaction(() => {
+                db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId);
+                deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)', {
+                    phase: 'verify-container', command: { state: 'accepted', message: 'Komodo accepted the request; recovery still requires verification.' },
+                    checkpoint: { key: 'container-action', label: 'Container command accepted' },
+                    blocker: { code: 'recovery-settling', message: 'Waiting for the recovery settling interval before checking health.', automatic: true }
+                });
+            })();
             return { kind: 'started' };
         }
         // These failures cannot have queued a remote restart. Other failures
         // (especially a timeout) may have: retain capacity until recovery.
-        if (result.kind === 'no-name' || result.kind === 'unconfigured') {
-            db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
-        } else {
-            db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId);
-        }
-        finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
+        db().transaction(() => {
+            if (result.kind === 'no-name' || result.kind === 'unconfigured') {
+                db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
+                recordProgress(operationId, { command: { state: 'rejected', message: 'The command could not be submitted: ' + result.message },
+                    recovery: { state: 'released', message: 'Capacity released because no remote command could be submitted.', operationId } });
+            } else {
+                db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId);
+                recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + result.message } });
+            }
+            finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
+        })();
         return { kind: 'failed', message: `komodo: ${result.message}` };
     } catch (err) {
         if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
         const message = `komodo: ${(err as Error).message}`;
-        db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId);
-        finishDeployment(operationId!, { ok: false, message });
+        db().transaction(() => {
+            db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId);
+            recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + (err as Error).message } });
+            finishDeployment(operationId!, { ok: false, message });
+        })();
         return { kind: 'failed', message };
     } finally {
         // Start the grace period at completion, including slow/failed requests.
@@ -145,31 +177,47 @@ async function refresh(): Promise<void> {
         if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
         let error: string | null = null;
         let players: number | null = null;
+        const checks: NonNullable<ProgressUpdate['checks']> = [];
         try {
             const health = await komodoInspectHealthy(slot.server);
+            checks.push({ key: 'container', label: 'Container readiness', result: health.ok ? 'pass' : 'waiting', source: 'komodo', actual: health.state ?? null,
+                checkedAt: new Date().toISOString(), message: health.ok ? null : health.reason });
             if (!health.ok) {
                 error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
             } else {
                 const status = await probeServerStatus(slot.server, slot.requested_at);
+                checks.push({ key: 'game', label: 'Game status', result: status.ok ? 'pass' : 'waiting', source: status.ok ? status.source : null,
+                    checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null,
+                    message: status.ok ? null : status.reason });
                 if (!status.ok) error = `rcon not answering and no fresh telemetry: ${status.reason}`;
                 else players = status.players;
             }
         } catch (err) {
             error = `health check: ${(err as Error).message}`;
+            checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', result: 'unknown', source: null, message: error });
         }
         const now = new Date().toISOString();
-        const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
-            WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
-            .run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
-        if (!checked.changes) continue;
-        if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) {
-            db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
-            if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
-            else {
-                deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)');
-                finishDeployment(slot.operation_id, { ok: true });
+        db().transaction(() => {
+            const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
+                WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
+                .run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
+            if (!checked.changes) return;
+            if (slot.operation_id) recordProgress(slot.operation_id, {
+                phase: checks.some(check => check.key === 'game') ? 'verify-game' : 'verify-container', checks, observedAt: now,
+                blocker: error ? { code: 'recovery-health', message: error, automatic: true } : null,
+                recovery: { state: error ? 'held' : 'released', at: now, operationId: slot.operation_id,
+                    message: error ? 'Restart capacity remains reserved while recovery is checked.' : 'Container and game recovery verified; restart capacity released.' },
+                checkpoints: error ? [] : [{ key: 'verify-container', label: 'Container readiness verified', at: now }, { key: 'verify-game', label: 'Game status verified', at: now }]
+            });
+            if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) {
+                db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
+                if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
+                else {
+                    deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)');
+                    finishDeployment(slot.operation_id, { ok: true });
+                }
             }
-        }
+        })();
     }
 }
 

src/lib/server/provisioning/worker.ts

@@ -5,10 +5,12 @@ import { getServer } from '../fleet/servers.js';
 import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
 import { processCreations, processDestructions, rebuildServer } from './instances.js';
 import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
-import { probePlayerCount } from '../fleet/status-evidence.js';
-import { refreshRestartRollout, restartInRollout, rolloutWaitReason } from './restarts.js';
+import { probePlayerEvidence } from '../fleet/status-evidence.js';
+import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js';
 import { cancelProvisioning } from './requests.js';
 import type { RuntimeAction } from '../../provisioning.js';
+import type { ProvisioningBlocker } from '../../provisioning-detail.js';
+import { recordProgress } from './progress.js';
 
 const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
 export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
@@ -18,7 +20,7 @@ const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM pro
 const priority = (sources: Source[]) => sources.some(s => s.kind === 'watchdog') ? 2 : sources.some(s => s.kind === 'beta') ? 0 : sources.some(s => s.kind === 'campaign') ? 1 : sources.some(s => s.kind === 'maintenance') ? 3 : 2;
 
 /** Source work may hold or withdraw a request, but never executes it. */
-function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled' {
+function sourceEligibility(operation: Operation, onHold?: (source: Source) => void): 'ready' | 'held' | 'cancelled' {
     const sources = sourcesFor(operation.id);
     if (!sources.length) return 'ready';
     // Watchdog intent gates every linked source, including its drift campaign.
@@ -26,7 +28,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
         if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(source.source_id, operation.server)
             || getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
         const incident = db().prepare('SELECT kind FROM watchdog_incidents WHERE id = ?').get(source.source_id) as { kind: 'health' | 'drift' };
-        if (!maintenanceAllowed(incident.kind)) return 'held';
+        if (!maintenanceAllowed(incident.kind)) { onHold?.(source); return 'held'; }
     }
     let held = false;
     let ready = false;
@@ -36,7 +38,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
                 .get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined;
             if (row?.provisioning_id !== operation.id || !row.files_changed) continue;
             if (['pending','failed'].includes(row.restart_state)) ready = true;
-            if (['held','manual'].includes(row.restart_state)) held = true;
+            if (['held','manual'].includes(row.restart_state)) { held = true; onHold?.(source); }
         } else if (source.kind === 'beta') {
             const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
                 LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL`)
@@ -44,12 +46,12 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
             if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue;
             if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue;
             if (!row.outcome || ['active','passed'].includes(row.outcome)) ready = true;
-            if (row.outcome === 'paused') held = true;
+            if (row.outcome === 'paused') { held = true; onHold?.(source); }
         } else if (source.kind === 'maintenance') {
             const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined;
             if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) {
                 if (getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
-                if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) return 'held';
+                if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) { onHold?.(source); return 'held'; }
                 ready = true;
             }
         } else ready = true;
@@ -57,26 +59,31 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
     return held ? 'held' : ready ? 'ready' : 'cancelled';
 }
 
-function blocker(server: string, id: string, action: RuntimeAction): string | null {
+function blocker(server: string, id: string, action: RuntimeAction): ProvisioningBlocker | null {
     const sources = sourcesFor(id);
     const rank = priority(sources);
     const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
         AND state IN ('queued','waiting','running','awaiting-health')`).all(server, id) as { id: string; state: string; action: string; rowid: number }[];
     const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { rowid: number };
-    if (others.some(other => other.state !== 'waiting' || other.action === 'destroy'
-        || priority(sourcesFor(other.id)) < rank || (priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid))) {
-        return 'Waiting for another provisioning operation on this server';
+    const conflict = others.find(other => other.state !== 'waiting' || other.action === 'destroy'
+        || priority(sourcesFor(other.id)) < rank || (priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid));
+    if (conflict) {
+        return { code: 'operation-conflict', message: 'Waiting for another provisioning operation on this server', automatic: true,
+            href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' };
     }
     // Beta activation must not wait on the campaign rows that the beta itself
     // holds. Campaign/maintenance callers likewise never block on their own job.
-    if (rank > 0 && db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
-        WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server';
-    if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL
+    const beta = rank > 0 ? db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
+        WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server) as { id: string } | undefined : undefined;
+    if (beta) return { code: 'beta-work', message: 'Waiting for beta work on this server', automatic: true, href: `/betas/${encodeURIComponent(beta.id)}`, label: 'Blocking beta' };
+    const campaign = rank > 1 ? db().prepare(`SELECT ds.deployment_id FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL
         AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS (
             SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health')))
         AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id)
-        LIMIT 1`).get(server, id)) return 'Waiting for campaign work on this server';
-    return rolloutWaitReason(server, id, action);
+        LIMIT 1`).get(server, id) as { deployment_id: string } | undefined : undefined;
+    if (campaign) return { code: 'campaign-work', message: 'Waiting for campaign work on this server', automatic: true,
+        href: `/campaigns/${encodeURIComponent(campaign.deployment_id)}`, label: 'Blocking campaign' };
+    return rolloutBlocker(server, id, action);
 }
 
 interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
@@ -100,7 +107,9 @@ async function run(): Promise<void> {
         try { await processOperation(operation); }
         catch (err) {
             // Observation failures are retried; remote mutations are never replayed.
-            deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message);
+            deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message, {
+                checks: [], blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true }
+            });
         }
     }
 }
@@ -112,36 +121,65 @@ async function processOperation(operation: Operation): Promise<void> {
     // Watchdog is paused; never strand the reserved replacement identity.
     if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
     if (operation.state === 'waiting') {
-        const eligibility = sourceEligibility(operation);
+        const heldBy: Source[] = [];
+        const eligibility = sourceEligibility(operation, source => heldBy.push(source));
         if (eligibility === 'cancelled') { cancelProvisioning(id, 'Requester cancelled or replaced this operation'); return; }
-        if (eligibility === 'held') { deploymentStage(id, 'Waiting for the requester to resume'); return; }
+        if (eligibility === 'held') {
+            const source = heldBy[0];
+            const href = source ? source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}`
+                : source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}`
+                : source.kind === 'watchdog' ? `/maintenance/${encodeURIComponent(source.source_id)}`
+                : `/maintenance?view=activity#job-${encodeURIComponent(source.source_id)}` : null;
+            deploymentStage(id, 'Waiting for the requester to resume', { phase: 'eligibility', checks: [],
+                blocker: { code: 'requester-held', message: 'Waiting for the requester to resume', automatic: false, href, label: 'Requesting work' } });
+            return;
+        }
     }
     if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
     const watchdog = sourcesFor(id).some(source => source.kind === 'watchdog');
-    if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; }
+    if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration', {
+        phase: 'eligibility', checks: [], blocker: { code: 'configuration', message: 'Waiting for Komodo and RCON configuration', automatic: false, href: '/settings', label: 'Integration settings' }
+    }); return; }
     if (operation.state === 'awaiting-health') {
         const health = await komodoInspectHealthy(server);
+        if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(id)) return;
+        recordProgress(id, { phase: 'verify-stopped', checks: [{ key: 'stopped', label: 'Container stopped', source: 'komodo',
+            result: isStopped(health.state) ? 'pass' : health.state ? 'waiting' : 'unknown', actual: health.state ?? null,
+            message: health.ok ? null : health.reason }], blocker: isStopped(health.state) ? null
+            : { code: 'stopped-state', message: 'Waiting for an explicit stopped container state.', automatic: true } });
         if (isStopped(health.state)) {
-            db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
-            deploymentStage(id, 'Verified container stopped');
-            finishDeployment(id, { ok: true });
+            db().transaction(() => {
+                db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
+                deploymentStage(id, 'Verified container stopped', { checkpoint: { key: 'verify-stopped', label: 'Stopped container state verified' } });
+                finishDeployment(id, { ok: true });
+            })();
         } else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
         return;
     }
     const reason = blocker(server, id, action);
-    if (reason) { deploymentStage(id, reason); return; }
+    if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; }
     const health = await komodoInspectHealthy(server);
     if (action !== 'start' && !isStopped(health.state)) {
-        const players = await probePlayerCount(server);
+        const evidence = await probePlayerEvidence(server);
+        if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
+        const players = evidence.players;
+        recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting',
+            source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }],
+            blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present',
+                message: players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`, automatic: true } });
         db().prepare('UPDATE server_deployments SET players = ?, checked_at = ? WHERE id = ?').run(players, new Date().toISOString(), id);
         if (players !== 0) {
             deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);
             return;
         }
+    } else if (action !== 'start') {
+        if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
+        recordProgress(id, { phase: 'eligibility', blocker: null, checks: [{ key: 'stopped', label: 'Stopped container permits disruption',
+            source: 'komodo', result: 'pass', actual: health.state ?? null }] });
     }
     // Requests can arrive during the probes. Recheck before claiming the action.
     const latestReason = blocker(server, id, action);
-    if (latestReason) { deploymentStage(id, latestReason); return; }
+    if (latestReason) { deploymentStage(id, latestReason.message, { phase: 'eligibility', blocker: latestReason }); return; }
     if (sourceEligibility(operation) !== 'ready') return;
     if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
     if (action === 'rebuild') { await rebuildServer(id); return; }
@@ -155,13 +193,26 @@ async function processOperation(operation: Operation): Promise<void> {
         }
         return;
     }
-    db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id);
-    deploymentStage(id, 'Stopping container');
+    db().transaction(() => {
+        db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id);
+        deploymentStage(id, 'Stopping container', { phase: 'container-action', blocker: null, checks: [],
+            checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' }, command: { state: 'attempted', message: 'Stop submission attempted.' } });
+    })();
     // Persist intent first; a lost reply must not issue another stop later.
     try {
         const result = await komodoRuntimeAction(server, 'stop');
-        if (!result.ok) { finishDeployment(id, { ok: false, message: result.message }); return; }
-        db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
-        deploymentStage(id, 'Waiting for container to stop');
-    } catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
+        if (!result.ok) {
+            recordProgress(id, { command: { state: result.kind === 'no-name' || result.kind === 'unconfigured' ? 'rejected' : 'uncertain', message: result.message } });
+            finishDeployment(id, { ok: false, message: result.message }); return;
+        }
+        db().transaction(() => {
+            db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
+            deploymentStage(id, 'Waiting for container to stop', { phase: 'verify-stopped',
+                checkpoint: { key: 'container-action', label: 'Stop command accepted' },
+                command: { state: 'accepted', message: 'Komodo accepted the stop request; stopped state still requires verification.' } });
+        })();
+    } catch (err) {
+        recordProgress(id, { command: { state: 'uncertain', message: (err as Error).message } });
+        finishDeployment(id, { ok: false, message: (err as Error).message });
+    }
 }