Preserve verified health when provisioning completes

src/lib/server/provisioning/health-handoff.ts

@@ -0,0 +1,37 @@
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
+import { monitoringIdentity } from '../fleet/health-observations.js';
+import { getServer, setServerFields, type ServerRow } from '../fleet/servers.js';
+import { latestGameSubmission, telemetryReport, type VerifiedGameStatus } from '../fleet/status-evidence.js';
+import { serverObservation } from '../fleet/status.js';
+import { activityFields, successfulStatusFields } from '../fleet/status-updates.js';
+
+/** Called inside the completion transaction; evidence must still describe this process. */
+export function handoffVerifiedHealth(server: string, identity: string,
+    container: { state: string; observedAt: string }, game: VerifiedGameStatus): boolean {
+    const current = getServer(server);
+    if (!current || current.desired_state !== 'running' || monitoringIdentity(server) !== identity) return false;
+    const fresh = (at: string) => {
+        const age = Date.now() - Date.parse(at);
+        return age >= 0 && age <= ONLINE_MAX_MS;
+    };
+    const gameAt = game.source === 'telemetry' ? game.receivedAt : game.observedAt;
+    const submission = latestGameSubmission(server);
+    if (!fresh(container.observedAt) || !fresh(gameAt) || (submission && gameAt <= submission)) return false;
+
+    const updates: Partial<ServerRow> = {};
+    if (!current.container_checked_at || current.container_checked_at <= container.observedAt) {
+        updates.container_state = container.state;
+        updates.container_checked_at = container.observedAt;
+    }
+    if (game.source === 'rcon' && (!current.last_checked_at || current.last_checked_at <= game.observedAt)) {
+        Object.assign(updates, successfulStatusFields(game.rcon, game.observedAt));
+    }
+    const candidate = { ...current, ...updates };
+    // Evaluate the health readers will see after the operation and recovery slot
+    // complete. Newer contradictory evidence must keep verification pending.
+    const status = serverObservation(candidate, { provisioning: false, settling: false,
+        submission, telemetry: telemetryReport(server) });
+    if (status.health !== 'online') return false;
+    setServerFields(server, { ...updates, ...activityFields(candidate, status) });
+    return true;
+}

src/lib/server/provisioning/native.ts

@@ -23,6 +23,8 @@ import { trackActivity } from '../pipeline/activity.js';
 import { fleetSettings } from '../settings.js';
 import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
 import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
+import { monitoringIdentity } from '../fleet/health-observations.js';
+import { handoffVerifiedHealth } from './health-handoff.js';
 import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js';
 import { recordScript, removeScript } from '../fleet/inventory.js';
 import { sha256Buffer } from '../fleet/files.js';
@@ -544,7 +546,9 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             recordProgress(job.id, { checks, ...(value.result === 'pass' ? { checkpoint: { key: phase, label: value.label } } : {}) });
         };
         deploymentStage(job.id, 'Checking container state with host agent', { phase: job.action === 'stop' ? 'verify-stopped' : 'verify-container', blocker: null, checks });
+        const verificationIdentity = monitoringIdentity(job.server);
         const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect');
+        const containerObservedAt = new Date().toISOString();
         if (!stillActive(job)) return;
         if (job.action === 'stop') {
             const stopped = ['stopped','exited','created','dead'].includes(health.state);
@@ -584,9 +588,15 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; }
             if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; }
         }
-        db().prepare('UPDATE servers SET container_state=?,container_checked_at=? WHERE name=?').run(health.state, new Date().toISOString(), job.server);
-        recordProgress(job.id, { blocker: null, checks });
-        complete(job); return;
+        const completed = db().transaction(() => {
+            if (!stillActive(job) || !handoffVerifiedHealth(job.server, verificationIdentity,
+                { state: health.state, observedAt: containerObservedAt }, status)) return false;
+            recordProgress(job.id, { blocker: null, checks });
+            complete(job);
+            return true;
+        })();
+        if (!completed) hold(job, 'Waiting for current health evidence before completing verification.');
+        return;
     }
     const reason = eligibility(job);
     if (reason) { hold(job, reason); return; }

src/lib/server/provisioning/restarts.ts

@@ -23,11 +23,24 @@ export function restartRecoveryPending(server: string): boolean {
     return slotFor(server)?.healthy_at === null;
 }
 
-/** The queued restart may still expose the previous healthy container. */
+type SettlingSlot = Pick<RestartSlot, 'requested_at' | 'healthy_at' | 'operation_id'>;
+interface SettlingOperation { id: string; action: string; state: string }
+
+/** Submitted lifecycle work stays Starting until verification ends, including slow restarts. */
+export function recoverySettling(slot: SettlingSlot | undefined, operation?: SettlingOperation | null): boolean {
+    if (!slot || slot.healthy_at) return false;
+    if (slot.operation_id) return operation?.id === slot.operation_id
+        && ['start', 'restart', 'rebuild'].includes(operation.action)
+        && ['running', 'awaiting-health'].includes(operation.state);
+    // Legacy recovery slots have no operation whose verification can be followed.
+    return Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS;
+}
+
 export function restartSettling(server: string): boolean {
     const slot = slotFor(server);
-    return !!(slot && !slot.healthy_at
-        && Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
+    const operation = slot?.operation_id ? db().prepare(`SELECT id, action, state FROM server_deployments
+        WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as SettlingOperation | undefined : undefined;
+    return recoverySettling(slot, operation);
 }
 
 export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null {