Preserve verified health when provisioning completes

src/lib/server/fleet/monitoring-snapshot.ts

@@ -8,6 +8,7 @@ import type { ServerRow } from './servers.js';
 import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
 import { ONLINE_MAX_MS } from '../../server-freshness.js';
 import { serverObservation } from './status.js';
+import { recoverySettling } from '../provisioning/restarts.js';
 
 const nativeFields = ['id','server','host_id','generation','library_id','image','bind_address','port','public_address','public_port','created_at','retired_at'];
 const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS instance_generation,n.library_id AS game_library_id,
@@ -21,21 +22,21 @@ const sql = `SELECT s.*, n.id AS native_instance_id,n.host_id,n.generation AS in
     (SELECT MAX(at) FROM (SELECT MAX(d.submitted_at) AS at FROM server_deployments d
         WHERE d.server=s.name AND d.server_deleted_at IS NULL AND d.action IN ('create','start','restart','rebuild')
         UNION ALL SELECT r.requested_at)) AS submission,
-    r.requested_at AS restart_at,r.healthy_at AS restart_healthy,t.received_at AS telemetry_at,t.payload AS telemetry_json
+    r.requested_at AS restart_at,r.healthy_at AS restart_healthy,r.operation_id AS restart_operation_id,t.received_at AS telemetry_at,t.payload AS telemetry_json
     FROM servers s LEFT JOIN native_instances n ON n.server=s.name AND n.retired_at IS NULL
     LEFT JOIN ec_hosts h ON h.id=n.host_id AND h.deleted_at IS NULL
     LEFT JOIN restart_rollout r ON r.server=s.name LEFT JOIN server_telemetry t ON t.server=s.name`;
 interface Row extends ServerRow {
     instance_json: string | null; host_inventory: string | null; host_heartbeat: string | null; credential_active: number;
     lifecycle_json: string | null; lifecycle_at: string | null; submission: string | null;
-    restart_at: string | null; restart_healthy: string | null; telemetry_at: string | null; telemetry_json: string | null;
+    restart_at: string | null; restart_healthy: string | null; restart_operation_id: string | null; telemetry_at: string | null; telemetry_json: string | null;
 }
 interface Record {
     server: ServerRow; native: NativeInstance | null;
     host: Pick<Host, 'connected' | 'connectionState' | 'inventory'> | null;
     lifecycle: { id: string; action: string; state: string; submitted_at: string | null } | null;
     lifecycleAt: string | null; submission: string | null; telemetry: GameTelemetry | null;
-    restartAt: string | null; restartHealthy: string | null;
+    restartAt: string | null; restartHealthy: string | null; restartOperationId: string | null;
 }
 
 /** A pass shares its reads; each asynchronous evidence boundary refreshes in one joined query. */
@@ -48,7 +49,7 @@ export class MonitoringSnapshot {
         if (server === undefined) this.records.clear(); else this.records.delete(server);
         for (const row of rows) {
             const { instance_json,host_inventory,host_heartbeat,credential_active,lifecycle_json,lifecycle_at,submission,
-                restart_at,restart_healthy,telemetry_at,telemetry_json,...fields } = row;
+                restart_at,restart_healthy,restart_operation_id,telemetry_at,telemetry_json,...fields } = row;
             const connected = !!credential_active && !!host_heartbeat && Date.now() - Date.parse(host_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
             let telemetry: GameTelemetry | null = null;
             if (telemetry_json) try {
@@ -60,7 +61,7 @@ export class MonitoringSnapshot {
                 host: host_inventory ? { connected, connectionState: connected ? 'online' : 'offline',
                     inventory: parseHostInventory(row.host_id!, host_inventory) } : null,
                 lifecycle: lifecycle_json ? JSON.parse(lifecycle_json) : null, lifecycleAt: lifecycle_at, submission, telemetry,
-                restartAt: restart_at, restartHealthy: restart_healthy });
+                restartAt: restart_at, restartHealthy: restart_healthy, restartOperationId: restart_operation_id });
         }
     }
     get(server: string) { return this.records.get(server); }
@@ -71,7 +72,8 @@ export class MonitoringSnapshot {
     }
     settling(server: string) {
         const record = this.get(server);
-        return !!record?.restartAt && !record.restartHealthy && Date.now() - Date.parse(record.restartAt) < 15_000;
+        return recoverySettling(record?.restartAt ? { requested_at: record.restartAt,
+            healthy_at: record.restartHealthy, operation_id: record.restartOperationId } : undefined, record?.lifecycle);
     }
     telemetryEvidence(server: string) {
         const record = this.get(server), report = record?.telemetry;

src/lib/server/fleet/status-evidence.ts

@@ -1,7 +1,7 @@
 import { db } from '../db/index.js';
 import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
 import { ONLINE_MAX_MS } from '../../server-freshness.js';
-import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
+import { rconGetPlayerCount, rconStatusResult, type RconStatusResult } from '../integrations/rcon.js';
 import { latestGameSubmission } from './lifecycle-evidence.js';
 
 export { latestGameSubmission } from './lifecycle-evidence.js';
@@ -59,9 +59,13 @@ export async function probePlayerEvidence(server: string): Promise<PlayerEvidenc
     return { players: null, source: null, observedAt: new Date().toISOString() };
 }
 
+export type VerifiedGameStatus = { ok: true; players: number; map: string | null; observedAt: string } & (
+    { source: 'rcon'; rcon: Extract<RconStatusResult, { ok: true }> }
+    | { source: 'telemetry'; receivedAt: string });
+
 /** Host agent readiness remains a caller prerequisite. Old-process telemetry is excluded. */
 export async function probeServerStatus(server: string, after?: string | null): Promise<
-    { ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry'; observedAt: string; receivedAt?: string } | { ok: false; reason: string; observedAt: string }
+    VerifiedGameStatus | { ok: false; reason: string; observedAt: string }
 > {
     const current = telemetryEvidence(server, after);
     if (current) return { ok: true, ...current, source: 'telemetry', observedAt: new Date().toISOString() };
@@ -70,7 +74,7 @@ export async function probeServerStatus(server: string, after?: string | null):
         const rcon = await rconStatusResult(server);
         const latest = telemetryEvidence(server, after);
         if (latest) return { ok: true, ...latest, source: 'telemetry', observedAt: new Date().toISOString() };
-        if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', observedAt: new Date().toISOString() };
+        if (rcon.ok) return { ok: true, players: rcon.status.players, map: rcon.status.map, source: 'rcon', rcon, observedAt: new Date().toISOString() };
         reason = rcon.reason;
     } catch (err) { reason = (err as Error).message; }
     const telemetry = telemetryEvidence(server, after);

src/lib/server/fleet/status-updates.ts

@@ -0,0 +1,22 @@
+import type { RconStatusResult } from '../integrations/rcon.js';
+import type { ServerObservation } from '../../server-observation.js';
+import type { ServerRow } from './servers.js';
+
+/** Monitoring and provisioning publish the same complete RCON observation. */
+export function successfulStatusFields(result: Extract<RconStatusResult, { ok: true }>, observedAt: string): Partial<ServerRow> {
+    return {
+        last_checked_at: observedAt, last_seen_at: observedAt,
+        last_players: result.status.players, last_map: result.status.map,
+        last_gametype: result.status.gametype, last_roster: JSON.stringify(result.status.roster),
+        last_reply_snippet: result.rawReply ?? null, last_error: null, rcon_failures: 0
+    };
+}
+
+/** Occupancy timers and match clearing follow the same source as the UI. */
+export function activityFields(current: ServerRow, status: ServerObservation): Partial<ServerRow> {
+    if (status.health !== 'online' || status.last_players === null) return {};
+    return status.last_players > 0
+        ? { active_since: current.active_since ?? status.last_seen_at, inactive_since: null }
+        : { active_since: null, inactive_since: current.inactive_since ?? status.last_seen_at,
+            game_started_at: null, game_ended_at: null };
+}

src/lib/server/fleet/status.ts

@@ -1,7 +1,7 @@
 import { rconHealthError, selectServerObservation, type GameObservation } from '../../server-observation.js';
 import { classifyHealth } from './health.js';
 import { isProvisioning } from './provisioning.js';
-import { restartSettling } from '../provisioning/restarts.js';
+import { restartSettling, recoverySettling } from '../provisioning/restarts.js';
 import { type ServerRow } from './servers.js';
 import { telemetryReport, latestGameSubmission } from './status-evidence.js';
 import { latestGameSubmissions } from './lifecycle-evidence.js';
@@ -13,11 +13,11 @@ interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTe
 /** One set of evidence reads per snapshot, rather than four queries per server. */
 export function observationReader() {
     const handle = db();
-    const operations = new Map((handle.prepare(`SELECT server, action, state FROM server_deployments
+    const operations = new Map((handle.prepare(`SELECT id, server, action, state FROM server_deployments
         WHERE rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE server_deleted_at IS NULL GROUP BY server)`).all() as
-        { server: string; action: string; state: string }[]).map(row => [row.server, row]));
-    const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at FROM restart_rollout').all() as
-        { server: string; requested_at: string; healthy_at: string | null }[]).map(row => [row.server, row]));
+        { id: string; server: string; action: string; state: string }[]).map(row => [row.server, row]));
+    const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at, operation_id FROM restart_rollout').all() as
+        { server: string; requested_at: string; healthy_at: string | null; operation_id: string | null }[]).map(row => [row.server, row]));
     const submissions = latestGameSubmissions();
     const reports = new Map<string, GameTelemetry>();
     for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as
@@ -32,7 +32,7 @@ export function observationReader() {
         const operation = operations.get(server.name), restart = restarts.get(server.name);
         return serverObservation(server, {
             provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state),
-            settling: !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
+            settling: recoverySettling(restart, operation),
             telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null
         });
     };

src/lib/server/pipeline/worker.ts

@@ -9,6 +9,7 @@ import type { RconFailure } from '../integrations/rcon.js';
 import { setServerFields, type ServerRow } from '../fleet/servers.js';
 import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, monitoringStatus, monitoringStatusWriter, monitoringIdentity, MonitoringChangedError, MonitoringSkippedError } from '../fleet/health-observations.js';
 import { serverObservation } from '../fleet/status.js';
+import { successfulStatusFields, activityFields } from '../fleet/status-updates.js';
 import { STATUS_REFRESH_MS } from '../../server-freshness.js';
 import { recordMetrics } from '../fleet/metrics.js';
 import { selectedZombieMap } from '../../map-aliases.js';
@@ -220,13 +221,7 @@ async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot,
                     updates.last_reply_snippet = result.rawReply ?? null;
                     if (result.ok) {
                         capacity = result.status.capacity ?? null;
-                        updates.last_players = result.status.players;
-                        updates.last_map = result.status.map;
-                        updates.last_gametype = result.status.gametype;
-                        updates.last_roster = JSON.stringify(result.status.roster);
-                        updates.last_seen_at = now;
-                        updates.last_error = null;
-                        updates.rcon_failures = 0;
+                        Object.assign(updates, successfulStatusFields(result, now));
                     } else {
                         // Only distinct accepted checks in the current lifecycle count.
                         // Skipped/superseded checks return above without changing the streak.
@@ -247,18 +242,7 @@ async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot,
     }
     const current = { ...(snapshot.get(srv.name)?.server ?? srv), ...updates } as ServerRow;
     const status = snapshot.observe(current);
-    // Occupancy timers and match clearing follow the same source as the UI.
-    if (status.health === 'online' && status.last_players !== null) {
-        if (status.last_players > 0) {
-            updates.active_since = current.active_since ?? status.last_seen_at;
-            updates.inactive_since = null;
-        } else {
-            updates.active_since = null;
-            updates.inactive_since = current.inactive_since ?? status.last_seen_at;
-            updates.game_started_at = null;
-            updates.game_ended_at = null;
-        }
-    }
+    Object.assign(updates, activityFields(current, status));
     flush();
     return capacity;
 }

src/lib/server/provisioning/health-handoff.ts

@@ -0,0 +1,37 @@
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
+import { monitoringIdentity } from '../fleet/health-observations.js';
+import { getServer, setServerFields, type ServerRow } from '../fleet/servers.js';
+import { latestGameSubmission, telemetryReport, type VerifiedGameStatus } from '../fleet/status-evidence.js';
+import { serverObservation } from '../fleet/status.js';
+import { activityFields, successfulStatusFields } from '../fleet/status-updates.js';
+
+/** Called inside the completion transaction; evidence must still describe this process. */
+export function handoffVerifiedHealth(server: string, identity: string,
+    container: { state: string; observedAt: string }, game: VerifiedGameStatus): boolean {
+    const current = getServer(server);
+    if (!current || current.desired_state !== 'running' || monitoringIdentity(server) !== identity) return false;
+    const fresh = (at: string) => {
+        const age = Date.now() - Date.parse(at);
+        return age >= 0 && age <= ONLINE_MAX_MS;
+    };
+    const gameAt = game.source === 'telemetry' ? game.receivedAt : game.observedAt;
+    const submission = latestGameSubmission(server);
+    if (!fresh(container.observedAt) || !fresh(gameAt) || (submission && gameAt <= submission)) return false;
+
+    const updates: Partial<ServerRow> = {};
+    if (!current.container_checked_at || current.container_checked_at <= container.observedAt) {
+        updates.container_state = container.state;
+        updates.container_checked_at = container.observedAt;
+    }
+    if (game.source === 'rcon' && (!current.last_checked_at || current.last_checked_at <= game.observedAt)) {
+        Object.assign(updates, successfulStatusFields(game.rcon, game.observedAt));
+    }
+    const candidate = { ...current, ...updates };
+    // Evaluate the health readers will see after the operation and recovery slot
+    // complete. Newer contradictory evidence must keep verification pending.
+    const status = serverObservation(candidate, { provisioning: false, settling: false,
+        submission, telemetry: telemetryReport(server) });
+    if (status.health !== 'online') return false;
+    setServerFields(server, { ...updates, ...activityFields(candidate, status) });
+    return true;
+}

src/lib/server/provisioning/native.ts

@@ -23,6 +23,8 @@ import { trackActivity } from '../pipeline/activity.js';
 import { fleetSettings } from '../settings.js';
 import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
 import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
+import { monitoringIdentity } from '../fleet/health-observations.js';
+import { handoffVerifiedHealth } from './health-handoff.js';
 import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js';
 import { recordScript, removeScript } from '../fleet/inventory.js';
 import { sha256Buffer } from '../fleet/files.js';
@@ -544,7 +546,9 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             recordProgress(job.id, { checks, ...(value.result === 'pass' ? { checkpoint: { key: phase, label: value.label } } : {}) });
         };
         deploymentStage(job.id, 'Checking container state with host agent', { phase: job.action === 'stop' ? 'verify-stopped' : 'verify-container', blocker: null, checks });
+        const verificationIdentity = monitoringIdentity(job.server);
         const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect');
+        const containerObservedAt = new Date().toISOString();
         if (!stillActive(job)) return;
         if (job.action === 'stop') {
             const stopped = ['stopped','exited','created','dead'].includes(health.state);
@@ -584,9 +588,15 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; }
             if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; }
         }
-        db().prepare('UPDATE servers SET container_state=?,container_checked_at=? WHERE name=?').run(health.state, new Date().toISOString(), job.server);
-        recordProgress(job.id, { blocker: null, checks });
-        complete(job); return;
+        const completed = db().transaction(() => {
+            if (!stillActive(job) || !handoffVerifiedHealth(job.server, verificationIdentity,
+                { state: health.state, observedAt: containerObservedAt }, status)) return false;
+            recordProgress(job.id, { blocker: null, checks });
+            complete(job);
+            return true;
+        })();
+        if (!completed) hold(job, 'Waiting for current health evidence before completing verification.');
+        return;
     }
     const reason = eligibility(job);
     if (reason) { hold(job, reason); return; }

src/lib/server/provisioning/restarts.ts

@@ -23,11 +23,24 @@ export function restartRecoveryPending(server: string): boolean {
     return slotFor(server)?.healthy_at === null;
 }
 
-/** The queued restart may still expose the previous healthy container. */
+type SettlingSlot = Pick<RestartSlot, 'requested_at' | 'healthy_at' | 'operation_id'>;
+interface SettlingOperation { id: string; action: string; state: string }
+
+/** Submitted lifecycle work stays Starting until verification ends, including slow restarts. */
+export function recoverySettling(slot: SettlingSlot | undefined, operation?: SettlingOperation | null): boolean {
+    if (!slot || slot.healthy_at) return false;
+    if (slot.operation_id) return operation?.id === slot.operation_id
+        && ['start', 'restart', 'rebuild'].includes(operation.action)
+        && ['running', 'awaiting-health'].includes(operation.state);
+    // Legacy recovery slots have no operation whose verification can be followed.
+    return Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS;
+}
+
 export function restartSettling(server: string): boolean {
     const slot = slotFor(server);
-    return !!(slot && !slot.healthy_at
-        && Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
+    const operation = slot?.operation_id ? db().prepare(`SELECT id, action, state FROM server_deployments
+        WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as SettlingOperation | undefined : undefined;
+    return recoverySettling(slot, operation);
 }
 
 export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null {

tests/unit/native-provisioning.test.ts

@@ -30,7 +30,7 @@ beforeEach(async () => {
         return { healthy: remote.state === 'running', state: remote.state, exists: true } as never;
     });
 });
-afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
+afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.unstubAllEnvs(); });
 
 async function prepareLibrary(game: 't5' | 't6' = 't5') {
     const libs = await import('../../src/lib/server/game-files/index.js');
@@ -74,6 +74,149 @@ async function detail(id: string) {
     return getDeployment(id)!.deployment;
 }
 
+
+async function observations(server: string) {
+    const { getServer } = await import('../../src/lib/server/fleet/servers.js');
+    const { serverObservation, observationReader } = await import('../../src/lib/server/fleet/status.js');
+    const { MonitoringSnapshot } = await import('../../src/lib/server/fleet/monitoring-snapshot.js');
+    const row = getServer(server)!;
+    return [serverObservation(row), observationReader()(row), new MonitoringSnapshot().observe(row)];
+}
+
+describe('provisioning health handoff', () => {
+    it.each(['create', 'start', 'restart', 'rebuild'] as const)('%s publishes verified game health before completion becomes visible', async action => {
+        const job = await queue(); await advance();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
+        let id = job.deploymentId;
+        if (action !== 'create') {
+            await acknowledge(id);
+            db().prepare('UPDATE servers SET container_state=? WHERE name=?').run(action === 'start' ? 'stopped' : 'running', job.name);
+            const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+            const request = requestLifecycle(job.name, action);
+            if (!request.ok) throw new Error(request.error);
+            id = request.deploymentId;
+            await advance();
+        }
+        const old = new Date(Date.now() - 120_000).toISOString();
+        db().prepare(`UPDATE servers SET last_seen_at=?,last_checked_at=?,last_players=0,last_roster='[]',
+            last_error='timeout:old process',rcon_failures=2,active_since=NULL,inactive_since=? WHERE name=?`).run(old, old, old, job.name);
+        vi.useFakeTimers({ toFake: ['Date'] });
+        vi.setSystemTime(Date.now() + 1000);
+        const observedAt = new Date().toISOString();
+        const roster = [{ num: 0, name: 'Player', guid: '12345678901234567', score: 500, ping: 20 }];
+        const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
+        vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: 'verified status',
+            status: { players: 1, map: remote.map, gametype: 'zom', roster } });
+        const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
+        const visible: string[] = [];
+        const unsubscribe = subscribe(event => {
+            if (event.type === 'fleet-invalidated' && getDeployment(id)?.deployment.state === 'done') {
+                visible.push(serverObservation(getServer(job.name)!).health);
+            }
+        });
+        try { await acknowledge(id); } finally { unsubscribe(); }
+        expect((await detail(id)).state).toBe('done');
+        expect(visible).toEqual(['online']);
+        expect(getServer(job.name)).toMatchObject({ last_checked_at: observedAt, last_seen_at: observedAt,
+            last_players: 1, last_roster: JSON.stringify(roster), last_reply_snippet: 'verified status',
+            last_error: null, rcon_failures: 0, active_since: observedAt, inactive_since: null });
+        for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', last_players: 1, roster });
+    });
+
+    it('completes directly from persisted telemetry without inventing an RCON observation', async () => {
+        const job = await queue(); await advance();
+        const operation = (await operationFor(job.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed', result: { state: 'running', healthy: true } });
+        vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000);
+        const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
+        const { gameEvent } = await import('../helpers/telemetry.js');
+        const receivedAt = new Date().toISOString();
+        saveTelemetry({ ...gameEvent('heartbeat', 0, 0, job.name), receivedAt });
+        await advance();
+        expect((await detail(job.deploymentId)).state).toBe('done');
+        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
+        expect(getServer(job.name)).toMatchObject({ last_seen_at: null, inactive_since: receivedAt });
+        for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', status_source: 'telemetry', last_seen_at: receivedAt });
+    });
+
+    it('rechecks expired verification evidence instead of renewing its timestamps', async () => {
+        const job = await queue(); await advance();
+        vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000);
+        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        vi.mocked(probeVersion).mockImplementationOnce(async () => {
+            vi.setSystemTime(Date.now() + 91_000);
+            heartbeat(hostId, inventory());
+            return 5354;
+        });
+        await acknowledge(job.deploymentId);
+        expect((await detail(job.deploymentId)).state).toBe('awaiting-health');
+        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
+        expect(getServer(job.name)?.last_seen_at).toBeNull();
+        for (const status of await observations(job.name)) expect(status.health).toBe('provisioning');
+        await advance();
+        expect((await detail(job.deploymentId)).state).toBe('done');
+        expect(getServer(job.name)?.last_seen_at).toBe(new Date().toISOString());
+    });
+
+    it.each(['instance', 'lifecycle'] as const)('discards verification when the %s changes during later checks', async change => {
+        const job = await queue(); await advance();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
+        vi.mocked(probeVersion).mockImplementationOnce(async () => {
+            if (change === 'instance') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
+            else db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() + 1000).toISOString(), job.deploymentId);
+            return 5354;
+        });
+        await acknowledge(job.deploymentId);
+        expect((await detail(job.deploymentId)).state).toBe('awaiting-health');
+        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
+        expect(getServer(job.name)).toMatchObject({ last_seen_at: null, container_checked_at: null });
+    });
+
+    it.each(['online', 'offline'] as const)('preserves newer %s monitoring evidence received during version verification', async health => {
+        const job = await queue(); await advance();
+        vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
+        let newer: string;
+        vi.mocked(probeVersion).mockImplementationOnce(async () => {
+            vi.setSystemTime(Date.now() + 1000); newer = new Date().toISOString();
+            db().prepare(`UPDATE servers SET container_state=?,container_checked_at=?,last_seen_at=?,last_checked_at=?,
+                last_players=2,last_roster='[]',last_error=NULL WHERE name=?`).run(health === 'online' ? 'running' : 'stopped', newer, newer, newer, job.name);
+            return 5354;
+        });
+        await acknowledge(job.deploymentId);
+        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
+        expect(getServer(job.name)).toMatchObject({ last_seen_at: newer!, container_checked_at: newer!, last_players: 2 });
+        expect((await detail(job.deploymentId)).state).toBe(health === 'online' ? 'done' : 'awaiting-health');
+        if (health === 'online') for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', last_players: 2 });
+    });
+
+    it('keeps a slow restart Starting until verification succeeds and surfaces terminal failures', async () => {
+        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const request = requestLifecycle(job.name, 'restart');
+        if (!request.ok) throw new Error(request.error);
+        await advance();
+        vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 20_000);
+        for (const status of await observations(job.name)) expect(status.health).toBe('starting');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        for (const state of ['failed', 'cancelled', 'interrupted']) {
+            db().prepare('UPDATE server_deployments SET state=? WHERE id=?').run(state, request.deploymentId);
+            for (const status of await observations(job.name)) expect(status.health).toBe('offline');
+        }
+        db().prepare("UPDATE server_deployments SET state='running' WHERE id=?").run(request.deploymentId);
+        await acknowledge(request.deploymentId);
+        expect((await detail(request.deploymentId)).state).toBe('done');
+        for (const status of await observations(job.name)) expect(status.health).toBe('online');
+    });
+});
+
 describe('native instance provisioning', () => {
     it('recognizes an unplaced native intent and resumes it once after a controller restart', async () => {
         const job = await queue();