Preserve verified health when provisioning completes

src/lib/server/fleet/monitoring-snapshot.ts

@@ -8,6 +8,7 @@ import type { ServerRow } from './servers.js';
 import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
 import { ONLINE_MAX_MS } from '../../server-freshness.js';
 import { serverObservation } from './status.js';
+import { recoverySettling } from '../provisioning/restarts.js';
 
 const nativeFields = ['id','server','host_id','generation','library_id','image','bind_address','port','public_address','public_port','created_at','retired_at'];
 const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS instance_generation,n.library_id AS game_library_id,
@@ -21,21 +22,21 @@ const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS in
     (SELECT MAX(at) FROM (SELECT MAX(d.submitted_at) AS at FROM server_deployments d
         WHERE d.server=s.name AND d.server_deleted_at IS NULL AND d.action IN ('create','start','restart','rebuild')
         UNION ALL SELECT r.requested_at)) AS submission,
-    r.requested_at AS restart_at,r.healthy_at AS restart_healthy,t.received_at AS telemetry_at,t.payload AS telemetry_json
+    r.requested_at AS restart_at,r.healthy_at AS restart_healthy,r.operation_id AS restart_operation_id,t.received_at AS telemetry_at,t.payload AS telemetry_json
     FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL
     LEFT JOIN ec_hosts h ON h.id=n.host_id AND h.deleted_at IS NULL
     LEFT JOIN restart_rollout r ON r.server=s.name LEFT JOIN server_telemetry t ON t.server=s.name`;
 interface Row extends ServerRow {
     instance_json: string | null; host_inventory: string | null; host_heartbeat: string | null; credential_active: number;
     lifecycle_json: string | null; lifecycle_at: string | null; submission: string | null;
-    restart_at: string | null; restart_healthy: string | null; telemetry_at: string | null; telemetry_json: string | null;
+    restart_at: string | null; restart_healthy: string | null; restart_operation_id: string | null; telemetry_at: string | null; telemetry_json: string | null;
 }
 interface Record {
     server: ServerRow; native: NativeInstance | null;
     host: Pick<Host, 'connected' | 'connectionState' | 'inventory'> | null;
     lifecycle: { id: string; action: string; state: string; submitted_at: string | null } | null;
     lifecycleAt: string | null; submission: string | null; telemetry: GameTelemetry | null;
-    restartAt: string | null; restartHealthy: string | null;
+    restartAt: string | null; restartHealthy: string | null; restartOperationId: string | null;
 }
 
 /** A pass shares its reads; each asynchronous evidence boundary refreshes in one joined query. */
@@ -48,7 +49,7 @@ export class MonitoringSnapshot {
         if (server === undefined) this.records.clear(); else this.records.delete(server);
         for (const row of rows) {
             const { instance_json,host_inventory,host_heartbeat,credential_active,lifecycle_json,lifecycle_at,submission,
-                restart_at,restart_healthy,telemetry_at,telemetry_json,...fields } = row;
+                restart_at,restart_healthy,restart_operation_id,telemetry_at,telemetry_json,...fields } = row;
             const connected = !!credential_active && !!host_heartbeat && Date.now() - Date.parse(host_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
             let telemetry: GameTelemetry | null = null;
             if (telemetry_json) try {
@@ -60,7 +61,7 @@ export class MonitoringSnapshot {
                 host: host_inventory ? { connected, connectionState: connected ? 'online' : 'offline',
                     inventory: parseHostInventory(row.host_id!, host_inventory) } : null,
                 lifecycle: lifecycle_json ? JSON.parse(lifecycle_json) : null, lifecycleAt: lifecycle_at, submission, telemetry,
-                restartAt: restart_at, restartHealthy: restart_healthy });
+                restartAt: restart_at, restartHealthy: restart_healthy, restartOperationId: restart_operation_id });
         }
     }
     get(server: string) { return this.records.get(server); }
@@ -71,7 +72,8 @@ export class MonitoringSnapshot {
     }
     settling(server: string) {
         const record = this.get(server);
-        return !!record?.restartAt && !record.restartHealthy && Date.now() - Date.parse(record.restartAt) < 15_000;
+        return recoverySettling(record?.restartAt ? { requested_at: record.restartAt,
+            healthy_at: record.restartHealthy, operation_id: record.restartOperationId } : undefined, record?.lifecycle);
     }
     telemetryEvidence(server: string) {
         const record = this.get(server), report = record?.telemetry;

src/lib/server/fleet/status-evidence.ts

@@ -1,7 +1,7 @@
 import { db } from '../db/index.js';
 import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
 import { ONLINE_MAX_MS } from '../../server-freshness.js';
-import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
+import { rconGetPlayerCount, rconStatusResult, type RconStatusResult } from '../integrations/rcon.js';
 import { latestGameSubmission } from './lifecycle-evidence.js';
 
 export { latestGameSubmission } from './lifecycle-evidence.js';
@@ -59,9 +59,13 @@ export async function probePlayerEvidence(server: string): Promise<PlayerEvidenc
     return { players: null, source: null, observedAt: new Date().toISOString() };
 }
 
+export type VerifiedGameStatus = { ok: true; players: number; map: string | null; observedAt: string } & (
+    { source: 'rcon'; rcon: Extract<RconStatusResult, { ok: true }> }
+    | { source: 'telemetry'; receivedAt: string });
+
 /** Host agent readiness remains a caller prerequisite. Old-process telemetry is excluded. */
 export async function probeServerStatus(server: string, after?: string | null): Promise<
-    { ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry'; observedAt: string; receivedAt?: string } | { ok: false; reason: string; observedAt: string }
+    VerifiedGameStatus | { ok: false; reason: string; observedAt: string }
 > {
     const current = telemetryEvidence(server, after);
     if (current) return { ok: true, ...current, source: 'telemetry', observedAt: new Date().toISOString() };
@@ -70,7 +74,7 @@ export async function probeServerStatus(server: string, after?: string | null):
         const rcon = await rconStatusResult(server);
         const latest = telemetryEvidence(server, after);
         if (latest) return { ok: true, ...latest, source: 'telemetry', observedAt: new Date().toISOString() };
-        if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', observedAt: new Date().toISOString() };
+        if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', rcon, observedAt: new Date().toISOString() };
         reason = rcon.reason;
     } catch (err) { reason = (err as Error).message; }
     const telemetry = telemetryEvidence(server, after);

src/lib/server/fleet/status-updates.ts

@@ -0,0 +1,22 @@
+import type { RconStatusResult } from '../integrations/rcon.js';
+import type { ServerObservation } from '../../server-observation.js';
+import type { ServerRow } from './servers.js';
+
+/** Monitoring and provisioning publish the same complete RCON observation. */
+export function successfulStatusFields(result: Extract<RconStatusResult, { ok: true }>, observedAt: string): Partial<ServerRow> {
+    return {
+        last_checked_at: observedAt, last_seen_at: observedAt,
+        last_players: result.status.players, last_map: result.status.map,
+        last_gametype: result.status.gametype, last_roster: JSON.stringify(result.status.roster),
+        last_reply_snippet: result.rawReply ?? null, last_error: null, rcon_failures: 0
+    };
+}
+
+/** Occupancy timers and match clearing follow the same source as the UI. */
+export function activityFields(current: ServerRow, status: ServerObservation): Partial<ServerRow> {
+    if (status.health !== 'online' || status.last_players === null) return {};
+    return status.last_players > 0
+        ? { active_since: current.active_since ?? status.last_seen_at, inactive_since: null }
+        : { active_since: null, inactive_since: current.inactive_since ?? status.last_seen_at,
+            game_started_at: null, game_ended_at: null };
+}

src/lib/server/fleet/status.ts

@@ -1,7 +1,7 @@
 import { rconHealthError, selectServerObservation, type GameObservation } from '../../server-observation.js';
 import { classifyHealth } from './health.js';
 import { isProvisioning } from './provisioning.js';
-import { restartSettling } from '../provisioning/restarts.js';
+import { restartSettling, recoverySettling } from '../provisioning/restarts.js';
 import { type ServerRow } from './servers.js';
 import { telemetryReport, latestGameSubmission } from './status-evidence.js';
 import { latestGameSubmissions } from './lifecycle-evidence.js';
@@ -13,11 +13,11 @@ interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTe
 /** One set of evidence reads per snapshot, rather than four queries per server. */
 export function observationReader() {
     const handle = db();
-    const operations = new Map((handle.prepare(`SELECT server, action, state FROM server_deployments
+    const operations = new Map((handle.prepare(`SELECT id, server, action, state FROM server_deployments
         WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`).all() as
-        { server: string; action: string; state: string }[]).map(row => [row.server, row]));
-    const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at FROM restart_rollout').all() as
-        { server: string; requested_at: string; healthy_at: string | null }[]).map(row => [row.server, row]));
+        { id: string; server: string; action: string; state: string }[]).map(row => [row.server, row]));
+    const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at, operation_id FROM restart_rollout').all() as
+        { server: string; requested_at: string; healthy_at: string | null; operation_id: string | null }[]).map(row => [row.server, row]));
     const submissions = latestGameSubmissions();
     const reports = new Map<string, GameTelemetry>();
     for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as
@@ -32,7 +32,7 @@ export function observationReader() {
         const operation = operations.get(server.name), restart = restarts.get(server.name);
         return serverObservation(server, {
             provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state),
-            settling: !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
+            settling: recoverySettling(restart, operation),
             telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null
         });
     };

src/lib/server/pipeline/worker.ts

@@ -9,6 +9,7 @@ import type { RconFailure } from '../integrations/rcon.js';
 import { setServerFields, type ServerRow } from '../fleet/servers.js';
 import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, monitoringStatus, monitoringStatusWriter, monitoringIdentity, MonitoringChangedError, MonitoringSkippedError } from '../fleet/health-observations.js';
 import { serverObservation } from '../fleet/status.js';
+import { successfulStatusFields, activityFields } from '../fleet/status-updates.js';
 import { STATUS_REFRESH_MS } from '../../server-freshness.js';
 import { recordMetrics } from '../fleet/metrics.js';
 import { selectedZombieMap } from '../../map-aliases.js';
@@ -220,13 +221,7 @@ async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot,
                     updates.last_reply_snippet = result.rawReply ?? null;
                     if (result.ok) {
                         capacity = result.status.capacity ?? null;
-                        updates.last_players = result.status.players;
-                        updates.last_map = result.status.map;
-                        updates.last_gametype = result.status.gametype;
-                        updates.last_roster = JSON.stringify(result.status.roster);
-                        updates.last_seen_at = now;
-                        updates.last_error = null;
-                        updates.rcon_failures = 0;
+                        Object.assign(updates, successfulStatusFields(result, now));
                     } else {
                         // Only distinct accepted checks in the current lifecycle count.
                         // Skipped/superseded checks return above without changing the streak.
@@ -247,18 +242,7 @@ async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot,
     }
     const current = { ...(snapshot.get(srv.name)?.server ?? srv), ...updates } as ServerRow;
     const status = snapshot.observe(current);
-    // Occupancy timers and match clearing follow the same source as the UI.
-    if (status.health === 'online' && status.last_players !== null) {
-        if (status.last_players > 0) {
-            updates.active_since = current.active_since ?? status.last_seen_at;
-            updates.inactive_since = null;
-        } else {
-            updates.active_since = null;
-            updates.inactive_since = current.inactive_since ?? status.last_seen_at;
-            updates.game_started_at = null;
-            updates.game_ended_at = null;
-        }
-    }
+    Object.assign(updates, activityFields(current, status));
     flush();
     return capacity;
 }

src/lib/server/provisioning/health-handoff.ts

@@ -0,0 +1,37 @@
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
+import { monitoringIdentity } from '../fleet/health-observations.js';
+import { getServer, setServerFields, type ServerRow } from '../fleet/servers.js';
+import { latestGameSubmission, telemetryReport, type VerifiedGameStatus } from '../fleet/status-evidence.js';
+import { serverObservation } from '../fleet/status.js';
+import { activityFields, successfulStatusFields } from '../fleet/status-updates.js';
+
+/** Called inside the completion transaction; evidence must still describe this process. */
+export function handoffVerifiedHealth(server: string, identity: string,
+    container: { state: string; observedAt: string }, game: VerifiedGameStatus): boolean {
+    const current = getServer(server);
+    if (!current || current.desired_state !== 'running' || monitoringIdentity(server) !== identity) return false;
+    const fresh = (at: string) => {
+        const age = Date.now() - Date.parse(at);
+        return age >= 0 && age <= ONLINE_MAX_MS;
+    };
+    const gameAt = game.source === 'telemetry' ? game.receivedAt : game.observedAt;
+    const submission = latestGameSubmission(server);
+    if (!fresh(container.observedAt) || !fresh(gameAt) || (submission && gameAt <= submission)) return false;
+
+    const updates: Partial<ServerRow> = {};
+    if (!current.container_checked_at || current.container_checked_at <= container.observedAt) {
+        updates.container_state = container.state;
+        updates.container_checked_at = container.observedAt;
+    }
+    if (game.source === 'rcon' && (!current.last_checked_at || current.last_checked_at <= game.observedAt)) {
+        Object.assign(updates, successfulStatusFields(game.rcon, game.observedAt));
+    }
+    const candidate = { ...current, ...updates };
+    // Evaluate the health readers will see after the operation and recovery slot
+    // complete. Newer contradictory evidence must keep verification pending.
+    const status = serverObservation(candidate, { provisioning: false, settling: false,
+        submission, telemetry: telemetryReport(server) });
+    if (status.health !== 'online') return false;
+    setServerFields(server, { ...updates, ...activityFields(candidate, status) });
+    return true;
+}

src/lib/server/provisioning/native.ts

@@ -23,6 +23,8 @@ import { trackActivity } from '../pipeline/activity.js';
 import { fleetSettings } from '../settings.js';
 import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
 import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
+import { monitoringIdentity } from '../fleet/health-observations.js';
+import { handoffVerifiedHealth } from './health-handoff.js';
 import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js';
 import { recordScript, removeScript } from '../fleet/inventory.js';
 import { sha256Buffer } from '../fleet/files.js';
@@ -544,7 +546,9 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             recordProgress(job.id, { checks, ...(value.result === 'pass' ? { checkpoint: { key: phase, label: value.label } } : {}) });
         };
         deploymentStage(job.id, 'Checking container state with host agent', { phase: job.action === 'stop' ? 'verify-stopped' : 'verify-container', blocker: null, checks });
+        const verificationIdentity = monitoringIdentity(job.server);
         const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect');
+        const containerObservedAt = new Date().toISOString();
         if (!stillActive(job)) return;
         if (job.action === 'stop') {
             const stopped = ['stopped','exited','created','dead'].includes(health.state);
@@ -584,9 +588,15 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; }
             if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; }
         }
-        db().prepare('UPDATE servers SET container_state=?,container_checked_at=? WHERE name=?').run(health.state, new Date().toISOString(), job.server);
-        recordProgress(job.id, { blocker: null, checks });
-        complete(job); return;
+        const completed = db().transaction(() => {
+            if (!stillActive(job) || !handoffVerifiedHealth(job.server, verificationIdentity,
+                { state: health.state, observedAt: containerObservedAt }, status)) return false;
+            recordProgress(job.id, { blocker: null, checks });
+            complete(job);
+            return true;
+        })();
+        if (!completed) hold(job, 'Waiting for current health evidence before completing verification.');
+        return;
     }
     const reason = eligibility(job);
     if (reason) { hold(job, reason); return; }

src/lib/server/provisioning/restarts.ts

@@ -23,11 +23,24 @@ export function restartRecoveryPending(server: string): boolean {
     return slotFor(server)?.healthy_at === null;
 }
 
-/** The queued restart may still expose the previous healthy container. */
+type SettlingSlot = Pick<RestartSlot, 'requested_at' | 'healthy_at' | 'operation_id'>;
+interface SettlingOperation { id: string; action: string; state: string }
+
+/** Submitted lifecycle work stays Starting until verification ends, including slow restarts. */
+export function recoverySettling(slot: SettlingSlot | undefined, operation?: SettlingOperation | null): boolean {
+    if (!slot || slot.healthy_at) return false;
+    if (slot.operation_id) return operation?.id === slot.operation_id
+        && ['start', 'restart', 'rebuild'].includes(operation.action)
+        && ['running', 'awaiting-health'].includes(operation.state);
+    // Legacy recovery slots have no operation whose verification can be followed.
+    return Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS;
+}
+
 export function restartSettling(server: string): boolean {
     const slot = slotFor(server);
-    return !!(slot && !slot.healthy_at
-        && Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
+    const operation = slot?.operation_id ? db().prepare(`SELECT id, action, state FROM server_deployments
+        WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as SettlingOperation | undefined : undefined;
+    return recoverySettling(slot, operation);
 }
 
 export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null {