Fix transient health checks and avoid unnecessary recovery

src/lib/server/fleet/lifecycle-evidence.ts

@@ -0,0 +1,8 @@
+import { db } from '../db/index.js';
+
+/** Reports from before the latest lifecycle submission describe the old process. */
+export function latestGameSubmission(server: string): string | null {
+    return (db().prepare(`SELECT MAX(at) AS at FROM (
+        SELECT submitted_at AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
+        UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
+}

src/lib/server/fleet/status-evidence.ts

@@ -2,16 +2,12 @@ import { db } from '../db/index.js';
 import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
 import { ONLINE_MAX_MS } from '../../server-freshness.js';
 import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
+import { latestGameSubmission } from './lifecycle-evidence.js';
+
+export { latestGameSubmission } from './lifecycle-evidence.js';
 
 export interface TelemetryEvidence { players: number; map: string; receivedAt: string }
 
-/** Reports from before the latest lifecycle submission describe the old process. */
-export function latestGameSubmission(server: string): string | null {
-    return (db().prepare(`SELECT MAX(at) AS at FROM (
-        SELECT submitted_at AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
-        UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
-}
-
 export function telemetryReport(server: string): GameTelemetry | null {
     const row = db().prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?').get(server) as
         { received_at: string; payload: string | null } | undefined;

src/lib/server/integrations/komodo.ts

@@ -320,7 +320,7 @@ const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']);
 async function fetchJsonWithSnippet(
     path: string,
     body: Record<string, string>
-): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string }> {
+): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string; unsupportedEndpoint?: boolean }> {
     const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
     let res: Response;
     try {
@@ -330,7 +330,11 @@ async function fetchJsonWithSnippet(
     }
     const text = await res.text().catch(() => '');
     if (!res.ok) {
-        return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text) };
+        const unsupportedEndpoint = res.status === 404 || (
+            [400, 405, 422, 500, 501].includes(res.status)
+            && /\b(?:unknown|unsupported|unrecognized)\s+(?:method|endpoint|variant)\b|\bmethod not found\b/i.test(text)
+        );
+        return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text), unsupportedEndpoint };
     }
     let json: unknown;
     try {
@@ -382,9 +386,10 @@ export async function komodoInspectHealthy(serverName: string): Promise<KomodoHe
     for (const { path, body } of attempts) {
         const r = await fetchJsonWithSnippet(path, body);
         if (!r.ok) {
+            // Alternate endpoints handle API-version differences. Retrying an
+            // outage against every endpoint delays all other server probes.
+            if (!r.unsupportedEndpoint) return { ok: false, reason: r.reason, snippet: r.snippet };
             lastSnippet = r.snippet ?? lastSnippet;
-            // Try the next endpoint on 404s or unknown-method-shaped errors; a
-            // real error (auth, timeout) will fail every attempt anyway.
             continue;
         }
         const state = extractContainerState(r.json);

src/lib/server/integrations/rcon.ts

@@ -2,6 +2,7 @@ import { createSocket } from 'node:dgram';
 import type { Socket } from 'node:dgram';
 import { config, komodoConfigured, rconConfigured } from '../config.js';
 import { getServer, setServerField } from '../fleet/servers.js';
+import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
 import { komodoResolveGamePort } from './komodo.js';
 import { parsePlutoniumRevision } from '../../plutonium.js';
 
@@ -300,13 +301,8 @@ export function parsePlayerRoster(text: string): RconPlayer[] {
 }
 
 export async function rconGetPlayerCount(server: string): Promise<number | null> {
-    const res = await endpointFor(server);
-    if (!res.endpoint) return null;
-    const send = await sendRcon(res.endpoint.host, res.endpoint.port, 'status');
-    if (!send.ok) return null;
-    const text = stripPrefix(send.buf);
-    if (!text) return null;
-    return parsePlayerCount(text);
+    const result = await rconStatusResult(server);
+    return result.ok ? result.status.players : null;
 }
 
 export interface RconStatus {
@@ -338,6 +334,18 @@ export type RconStatusResult =
       };
 
 const RAW_REPLY_LIMIT = 512;
+const statusInFlight = new Map<string, Promise<RconStatusResult>>();
+
+/** Retry only this read-only query: commands with effects must never be replayed. */
+async function sendStatus(endpoint: RconEndpoint): Promise<SendResult> {
+    for (let attempt = 0; ; attempt++) {
+        const result = await sendRcon(endpoint.host, endpoint.port, 'status');
+        if (result.ok || (result.reason !== 'timeout' && result.reason !== 'send-error') || attempt === 2) {
+            return result;
+        }
+        await new Promise<void>(resolve => setTimeout(resolve, 250 * 2 ** attempt));
+    }
+}
 
 /**
  * Render a Buffer as a short hex + printable-ASCII snippet — same layout as
@@ -365,6 +373,10 @@ export function formatRawReply(buf: Buffer): string {
  * rather than just "unreachable".
  */
 export async function rconStatusResult(server: string): Promise<RconStatusResult> {
+    // Snapshot before any network work. A probe begun after a restart or a
+    // replacement registration must not join one observing the old process.
+    const createdAt = getServer(server)?.created_at;
+    const epoch = createdAt ? latestGameSubmission(server) : null;
     const res = await endpointFor(server);
     if (!res.endpoint) {
         if (res.reason === 'unknown-server') {
@@ -387,13 +399,22 @@ export async function rconStatusResult(server: string): Promise<RconStatusResult
     }
 
     const ep = res.endpoint;
+    const key = JSON.stringify([server, createdAt, epoch, ep.host, ep.port]);
+    const pending = statusInFlight.get(key);
+    if (pending) return pending;
+    const request = probeStatus(ep).finally(() => { statusInFlight.delete(key); });
+    statusInFlight.set(key, request);
+    return request;
+}
+
+async function probeStatus(ep: RconEndpoint): Promise<RconStatusResult> {
     // Plutonium T5 doesn't recognize `getstatus` as an rcon subcommand
     // ("Unknown command"), so we use `status` — a print-body response with
     // a header table plus one row per player. Q3-derived engines that
     // *do* answer statusResponse (via anonymous `getstatus` query) would
     // give richer info-line dvars, but every engine we care about supports
     // `status` under the rcon wrapper.
-    const send = await sendRcon(ep.host, ep.port, 'status');
+    const send = await sendStatus(ep);
     if (!send.ok) {
         return { ok: false, endpoint: ep, reason: send.reason, detail: send.detail };
     }

src/lib/server/maintenance/recovery.ts

@@ -5,9 +5,11 @@ import { getServer, listServers } from '../fleet/servers.js';
 import { hasActiveProvisioning } from '../fleet/provisioning.js';
 import { komodoInspectHealthy } from '../integrations/komodo.js';
 import { rconStatusResult } from '../integrations/rcon.js';
-import { telemetryEvidence } from '../fleet/status-evidence.js';
+import { latestGameSubmission, telemetryEvidence, telemetryReport } from '../fleet/status-evidence.js';
 import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '../fleet/drift.js';
-import { requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
+import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
+import { deploymentStage, finishDeployment } from '../provisioning/history.js';
+import { recordProgress } from '../provisioning/progress.js';
 import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident, WatchdogOverview, WatchdogState } from '../../watchdog.js';
 
 import { maintenanceAllowed, setMaintenancePolicy } from './policies.js';
@@ -43,7 +45,13 @@ function detect(server: string, kind: 'health' | 'drift', summary: string, detai
         const newSample = Date.now() - Date.parse(previous.last_seen_at) >= WATCHDOG_SCAN_MS;
         const actionable = (details as WatchdogEvidence & { actionable?: boolean }).actionable;
         const priorActionable = (JSON.parse(previous.details) as { actionable?: boolean }).actionable;
-        const reset = kind === 'health' && actionable === true && priorActionable !== true;
+        // A successful routine/manual probe between watchdog samples breaks
+        // the failure streak, even if our next probe happens to fail again.
+        const after = latestGameSubmission(server);
+        const successful = [getServer(server)?.last_seen_at, telemetryReport(server)?.receivedAt]
+            .some(at => at && Date.parse(at) > Date.parse(previous.last_seen_at)
+                && Date.parse(at) <= Date.now() && (!after || Date.parse(at) > Date.parse(after)));
+        const reset = kind === 'health' && (successful || (actionable === true && priorActionable !== true));
         db().prepare('UPDATE watchdog_incidents SET details=?, last_seen_at=?, failures=?, first_seen_at=? WHERE id=?')
             .run(JSON.stringify(details), newSample || reset ? iso() : previous.last_seen_at,
                 reset ? 1 : previous.failures + (newSample ? 1 : 0), reset ? iso() : previous.first_seen_at, previous.id);
@@ -56,6 +64,43 @@ function detect(server: string, kind: 'health' | 'drift', summary: string, detai
     event(id, summary);
     return incident(id)!;
 }
+
+/** Call only after verifying container readiness and current game evidence. */
+export function resolveRecoveredHealthOperation(operationId: string, players: number, source: 'rcon' | 'telemetry' | null): boolean {
+    return db().transaction(() => {
+        const operation = db().prepare(`SELECT d.server, d.action, d.state, d.submitted_at, d.triggered_by, i.id AS incident_id
+            FROM server_deployments d JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
+            JOIN watchdog_incidents i ON i.id = s.source_id AND i.server = d.server
+            WHERE d.id = ? AND d.server_deleted_at IS NULL AND i.server_deleted_at IS NULL
+                AND i.kind = 'health' AND i.state IN ('open','recovering')
+                AND NOT EXISTS (SELECT 1 FROM provisioning_sources other WHERE other.operation_id = d.id AND other.kind != 'watchdog')`)
+            .get(operationId) as { server: string; action: string; state: string; submitted_at: string | null; triggered_by: string; incident_id: string } | undefined;
+        if (!operation || getServer(operation.server)?.desired_state !== 'running') return false;
+        const waiting = operation.state === 'waiting' && operation.submitted_at === null;
+        if (waiting && !operation.triggered_by.startsWith('watchdog:')) return false;
+        // Cancelling our unsubmitted request is safe when later queued work
+        // exists, and lets that work proceed. Submitted recovery must retain
+        // exclusive ownership before accepting evidence or releasing its slot.
+        if (!waiting && hasActiveProvisioning(operation.server, operationId)) return false;
+        if (!waiting && (operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state))) return false;
+        const message = waiting ? 'Server recovered before automatic recovery was submitted' : 'Fresh container and game health verified after restart';
+        if (waiting && !cancelProvisioning(operationId, message)) return false;
+        const now = iso();
+        db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = NULL
+            WHERE server = ? AND operation_id = ? AND healthy_at IS NULL`).run(now, now, players, operation.server, operationId);
+        db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, operationId);
+        recordProgress(operationId, { blocker: null, observedAt: now,
+            recovery: { state: 'released', at: now, operationId, message },
+            checks: [{ key: 'container', label: 'Container readiness', source: 'komodo', result: 'pass', checkedAt: now },
+                { key: 'game', label: 'Game status', source, result: 'pass', actual: players, checkedAt: now }] });
+        if (operation.state === 'awaiting-health') {
+            deploymentStage(operationId, message);
+            finishDeployment(operationId, { ok: true });
+        }
+        transition(operation.incident_id, 'resolved', message);
+        return true;
+    })();
+}
 function actions(id: string): WatchdogAction[] {
     return db().prepare(`SELECT a.*, d.state AS operation_state, d.stage AS operation_stage, d.error AS operation_error
         FROM watchdog_actions a LEFT JOIN server_deployments d ON d.id=a.operation_id
@@ -157,6 +202,15 @@ async function reconcileActions(): Promise<void> {
         if (!maintenanceAllowed(row.kind)) continue;
         const current = getServer(row.server);
         if (!current || current.desired_state === 'stopped' || protectedWork(row.server)) continue;
+        // A recovery deadline is a reason to check again, not evidence that
+        // rebuilding is necessary. The worker may not have polled this tick yet.
+        const observation = await observe(row.server, action.operation_id);
+        if (!observation || incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue;
+        if (observation.healthy) {
+            resolveRecoveredHealthOperation(action.operation_id, observation.players!, observation.source ?? null);
+            continue;
+        }
+        if (!observation.actionable) continue;
         const result = db().transaction(() => {
             const next = escalateWatchdogRestart(row.id, action.operation_id, WATCHDOG_RECOVERY_MS);
             if (next.ok) addAction(row, next.deploymentId, 'rebuild');
@@ -166,16 +220,24 @@ async function reconcileActions(): Promise<void> {
     }
 }
 
-interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null }
-async function observe(server: string): Promise<Observation | null> {
+interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; players?: number; source?: 'rcon' | 'telemetry' }
+async function observe(server: string, operationId?: string): Promise<Observation | null> {
+    const epoch = latestGameSubmission(server);
+    const suppressed = () => (operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server))
+        || protectedWork(server) || getServer(server)?.desired_state !== 'running' || epoch !== latestGameSubmission(server);
     const health = await komodoInspectHealthy(server);
-    if (scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return null;
+    if (suppressed()) return null;
     const state = health.state?.toLowerCase() ?? null;
     if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
-    if (telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state };
+    const telemetry = telemetryEvidence(server);
+    if (telemetry) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
     const rcon = await rconStatusResult(server);
-    if (rcon.ok || telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state };
-    return { healthy: false, actionable: true, reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
+    if (suppressed()) return null;
+    const latest = telemetryEvidence(server);
+    if (rcon.ok || latest) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state,
+        players: latest?.players ?? (rcon.ok ? rcon.status.players : 0), source: latest ? 'telemetry' : 'rcon' };
+    return { healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
+        reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
 }
 async function scanServer(server: string): Promise<void> {
     const row = getServer(server);

src/lib/server/pipeline/worker.ts

@@ -6,7 +6,7 @@ import { emit } from './bus.js';
 import type { ServerStatusEntry } from './bus.js';
 import { rconStatusResult } from '../integrations/rcon.js';
 import type { RconFailure } from '../integrations/rcon.js';
-import { getServer, listServers, setServerField } from '../fleet/servers.js';
+import { getServer, listServers, setServerField, type ServerRow } from '../fleet/servers.js';
 import { komodoInspectHealthy } from '../integrations/komodo.js';
 import { restartSettling } from '../provisioning/index.js';
 import { serverObservation } from '../fleet/status.js';
@@ -58,92 +58,126 @@ function encodeError(reason: RconFailure, detail: string): string {
     return `${reason}:${detail}`;
 }
 
-let statusInFlight: Promise<void> | null = null;
+// Deduplicate queued/running checks per server. A slow server must not prevent
+// completed servers from being polled again on the next 30-second tick.
+const pendingStatus = new Map<string, Promise<number | null>>();
+const statusQueue: Array<() => Promise<void>> = [];
+let activeStatus = 0;
 
-export function fanoutStatus(): Promise<void> {
-    if (statusInFlight) return statusInFlight;
-    statusInFlight = refreshStatus().finally(() => { statusInFlight = null; });
-    return statusInFlight;
+function drainStatusQueue(): void {
+    while (activeStatus < STATUS_CONCURRENCY && statusQueue.length) {
+        const next = statusQueue.shift()!;
+        activeStatus++;
+        void next().finally(() => { activeStatus--; drainStatusQueue(); });
+    }
 }
 
-async function refreshStatus(): Promise<void> {
+function queueStatus(server: string): Promise<number | null> {
+    const pending = pendingStatus.get(server);
+    if (pending) return pending;
+    let finish!: (capacity: number | null) => void;
+    const work = new Promise<number | null>(resolve => { finish = resolve; });
+    pendingStatus.set(server, work);
+    statusQueue.push(async () => {
+        let capacity: number | null = null;
+        try {
+            capacity = await refreshServerStatus(server);
+            // Consumers expect a full snapshot. Re-read it now so neither game
+            // data nor container evidence waits for the slowest fleet member.
+            emit({ type: 'server-status', entries: listServers().map(statusEntry) });
+        } catch (err) { console.error(`[worker] status error for ${server}:`, err); }
+        finally { pendingStatus.delete(server); finish(capacity); }
+    });
+    drainStatusQueue();
+    return work;
+}
+
+export async function fanoutStatus(): Promise<void> {
     const servers = listServers();
     if (!servers.length) return;
-    const entries: ServerStatusEntry[] = new Array(servers.length);
-    const capacities = new Map<string, number | null>();
-    let cursor = 0;
-    async function pump() {
-        while (cursor < servers.length) {
-            const i = cursor++;
-            const srv = servers[i];
-            if (!isProvisioning(srv.name)) {
-                const epoch = latestGameSubmission(srv.name);
-                const container = restartSettling(srv.name)
-                    ? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
-                    : await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message }));
-                // A lifecycle request may be submitted while an inspection is in flight.
-                if (epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)) {
-                    setServerField(srv.name, 'container_state', container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown');
-                    setServerField(srv.name, 'container_checked_at', new Date().toISOString());
-                    if (!container.ok) {
-                        setServerField(srv.name, 'last_error', null);
-                        setServerField(srv.name, 'last_reply_snippet', null);
-                    } else if (!telemetryEvidence(srv.name)) {
-                        const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
-                            reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));
-                        if (epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)) {
-                            const now = new Date().toISOString();
-                            setServerField(srv.name, 'last_checked_at', now);
-                            setServerField(srv.name, 'last_reply_snippet', result.rawReply ?? null);
-                            if (result.ok) {
-                                capacities.set(srv.name, result.status.capacity ?? null);
-                                setServerField(srv.name, 'last_players', result.status.players);
-                                setServerField(srv.name, 'last_map', result.status.map);
-                                setServerField(srv.name, 'last_gametype', result.status.gametype);
-                                setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster));
-                                setServerField(srv.name, 'last_seen_at', now);
-                                setServerField(srv.name, 'last_error', null);
-                            } else setServerField(srv.name, 'last_error', encodeError(result.reason, result.detail));
-                        }
-                    }
-                }
-            } else {
+    const results = await Promise.all(servers.map(server => queueStatus(server.name)));
+    const capacities = new Map(servers.map((server, i) => [server.name, results[i]]));
+    try {
+        // Metrics remain complete fleet collections, using freshness at capture
+        // time rather than the status saved at the beginning of a long scan.
+        recordMetrics(listServers().map(srv => {
+            const current = serverObservation(srv);
+            return { server: srv.name, game: srv.game ?? 'unknown',
+                map: selectedZombieMap(srv.game, current.last_map ?? srv.map, srv.map_slug)?.friendly ?? current.last_map ?? srv.map ?? 'Unknown map',
+                players: current.health === 'online' ? current.last_players : null,
+                capacity: capacities.get(srv.name) ?? null };
+        }));
+    } catch (err) { console.error('[worker] metrics error:', err); }
+}
+
+async function refreshServerStatus(server: string): Promise<number | null> {
+    const srv = getServer(server);
+    if (!srv) return null;
+    let capacity: number | null = null;
+    if (!isProvisioning(srv.name)) {
+        const epoch = latestGameSubmission(srv.name);
+        const unchanged = () => epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)
+            && getServer(srv.name)?.created_at === srv.created_at;
+        const container = restartSettling(srv.name)
+            ? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
+            : await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message }));
+        // A lifecycle request may be submitted while an inspection is in flight.
+        if (unchanged()) {
+            setServerField(srv.name, 'container_state', container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown');
+            setServerField(srv.name, 'container_checked_at', new Date().toISOString());
+            if (!container.ok) {
                 setServerField(srv.name, 'last_error', null);
                 setServerField(srv.name, 'last_reply_snippet', null);
-            }
-            const current = getServer(srv.name) ?? srv;
-            const status = serverObservation(current);
-            // Occupancy timers and match clearing follow the same source as the UI.
-            if (status.health === 'online' && status.last_players !== null) {
-                if (status.last_players > 0) {
-                    setServerField(srv.name, 'active_since', current.active_since ?? status.last_seen_at);
-                    setServerField(srv.name, 'inactive_since', null);
-                } else {
-                    setServerField(srv.name, 'active_since', null);
-                    setServerField(srv.name, 'inactive_since', current.inactive_since ?? status.last_seen_at);
-                    setServerField(srv.name, 'game_started_at', null);
-                    setServerField(srv.name, 'game_ended_at', null);
+            } else if (!telemetryEvidence(srv.name)) {
+                const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
+                    reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));
+                if (unchanged()) {
+                    const now = new Date().toISOString();
+                    setServerField(srv.name, 'last_checked_at', now);
+                    setServerField(srv.name, 'last_reply_snippet', result.rawReply ?? null);
+                    if (result.ok) {
+                        capacity = result.status.capacity ?? null;
+                        setServerField(srv.name, 'last_players', result.status.players);
+                        setServerField(srv.name, 'last_map', result.status.map);
+                        setServerField(srv.name, 'last_gametype', result.status.gametype);
+                        setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster));
+                        setServerField(srv.name, 'last_seen_at', now);
+                        setServerField(srv.name, 'last_error', null);
+                    } else setServerField(srv.name, 'last_error', encodeError(result.reason, result.detail));
                 }
             }
-            const final = serverObservation(getServer(srv.name) ?? current);
-            entries[i] = { server: srv.name, players: final.last_players, map: final.last_map ?? srv.map,
-                gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at,
-                lastError: final.last_error, replySnippet: final.replySnippet,
-                endpoint: current.game_port ? { host: current.rcon_host ?? config.rcon.host, port: current.game_port } : null, reason: final.last_error ? final.last_error.split(':')[0] as RconFailure : null,
-                activeSince: final.active_since, inactiveSince: final.inactive_since,
-                gameStartedAt: final.game_started_at, gameEndedAt: final.game_ended_at,
-                containerState: final.container_state, containerCheckedAt: final.container_checked_at,
-                telemetryAfter: final.telemetry_after, rconStatus: final.rcon_status, telemetry: final.telemetry };
+        }
+    } else {
+        setServerField(srv.name, 'last_error', null);
+        setServerField(srv.name, 'last_reply_snippet', null);
+    }
+    const current = getServer(srv.name) ?? srv;
+    const status = serverObservation(current);
+    // Occupancy timers and match clearing follow the same source as the UI.
+    if (status.health === 'online' && status.last_players !== null) {
+        if (status.last_players > 0) {
+            setServerField(srv.name, 'active_since', current.active_since ?? status.last_seen_at);
+            setServerField(srv.name, 'inactive_since', null);
+        } else {
+            setServerField(srv.name, 'active_since', null);
+            setServerField(srv.name, 'inactive_since', current.inactive_since ?? status.last_seen_at);
+            setServerField(srv.name, 'game_started_at', null);
+            setServerField(srv.name, 'game_ended_at', null);
         }
     }
-    await Promise.all(Array.from({ length: Math.min(STATUS_CONCURRENCY, servers.length) }, pump));
-    try {
-        recordMetrics(servers.map((srv, i) => ({ server: srv.name, game: srv.game ?? 'unknown',
-            map: selectedZombieMap(srv.game, entries[i].map ?? srv.map, srv.map_slug)?.friendly ?? entries[i].map ?? srv.map ?? 'Unknown map',
-            players: entries[i].health === 'online' ? entries[i].players : null,
-            capacity: capacities.get(srv.name) ?? null })));
-    } catch (err) { console.error('[worker] metrics error:', err); }
-    emit({ type: 'server-status', entries });
+    return capacity;
+}
+
+function statusEntry(srv: ServerRow): ServerStatusEntry {
+    const final = serverObservation(srv);
+    return { server: srv.name, players: final.last_players, map: final.last_map ?? srv.map,
+        gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at,
+        lastError: final.last_error, replySnippet: final.replySnippet,
+        endpoint: srv.game_port ? { host: srv.rcon_host ?? config.rcon.host, port: srv.game_port } : null, reason: final.last_error ? final.last_error.split(':')[0] as RconFailure : null,
+        activeSince: final.active_since, inactiveSince: final.inactive_since,
+        gameStartedAt: final.game_started_at, gameEndedAt: final.game_ended_at,
+        containerState: final.container_state, containerCheckedAt: final.container_checked_at,
+        telemetryAfter: final.telemetry_after, rconStatus: final.rcon_status, telemetry: final.telemetry };
 }
 
 export function startWorker(): void {

src/lib/server/provisioning/requests.ts

@@ -139,12 +139,21 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
     return db().transaction(() => {
         const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;
         if (blocker) return { ok: false as const, status: 409, error: blocker };
-        const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
+        const existing = db().prepare(`SELECT id, action, state, submitted_at, triggered_by FROM server_deployments WHERE server = ?
             AND server_deleted_at IS NULL
-            AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
+            AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as
+            { id: string; action: string; state: string; submitted_at: string | null; triggered_by: string } | undefined;
         if (existing) {
             if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
             db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
+            if (existing.state === 'waiting' && !existing.submitted_at && existing.triggered_by.startsWith('watchdog:')
+                && db().prepare(`SELECT 1 FROM provisioning_sources s JOIN watchdog_incidents i ON i.id = s.source_id
+                    WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health'`).get(existing.id)) {
+                // An operator explicitly requesting this queued action keeps
+                // that intent even if the original health condition clears.
+                db().prepare("UPDATE server_deployments SET triggered_by = 'ui' WHERE id = ?").run(existing.id);
+                recordDeploymentEvent(existing.id, 'Operator explicitly requested the queued health recovery action');
+            }
             return { ok: true as const, deploymentId: existing.id };
         }
         const row = getServer(server);

src/lib/server/provisioning/worker.ts

@@ -1,11 +1,12 @@
 import { maintenanceAllowed } from '../maintenance/policies.js';
+import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
 import { db } from '../db/index.js';
 import { komodoConfigured, rconConfigured } from '../config.js';
 import { getServer } from '../fleet/servers.js';
 import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
 import { processCreations, processDestructions, rebuildServer } from './instances.js';
 import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
-import { probePlayerEvidence } from '../fleet/status-evidence.js';
+import { latestGameSubmission, probePlayerEvidence } from '../fleet/status-evidence.js';
 import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js';
 import { cancelProvisioning } from './requests.js';
 import type { RuntimeAction } from '../../provisioning.js';
@@ -158,11 +159,28 @@ async function processOperation(operation: Operation): Promise<void> {
     }
     const reason = blocker(server, id, action);
     if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; }
+    const healthRecovery = watchdog && !!db().prepare(`SELECT 1 FROM provisioning_sources s
+        JOIN watchdog_incidents i ON i.id = s.source_id JOIN server_deployments d ON d.id = s.operation_id
+        WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health' AND d.triggered_by LIKE 'watchdog:%'`).get(id);
+    const epoch = latestGameSubmission(server);
     const health = await komodoInspectHealthy(server);
+    if (healthRecovery && epoch !== latestGameSubmission(server)) return;
+    if (healthRecovery && !health.ok && !['stopped','exited','created','dead','unhealthy','starting','restarting'].includes(health.state?.toLowerCase() ?? '')) {
+        deploymentStage(id, 'Waiting for current container health before automatic recovery', { phase: 'eligibility', checks: [],
+            blocker: { code: 'observation-unavailable', message: health.reason, automatic: true } });
+        return;
+    }
     if (action !== 'start' && !isStopped(health.state)) {
         const evidence = await probePlayerEvidence(server);
         if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
         const players = evidence.players;
+        if (healthRecovery && epoch !== latestGameSubmission(server)) return;
+        if (healthRecovery && health.ok && players !== null) {
+            // The fresh occupancy check also proves that the game recovered.
+            // Empty is not a reason to execute an obsolete health restart.
+            resolveRecoveredHealthOperation(id, players, evidence.source);
+            return;
+        }
         recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting',
             source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }],
             blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present',

src/routes/api/servers/[name]/probe-rcon/+server.ts

@@ -2,6 +2,8 @@ import { json } from '@sveltejs/kit';
 import type { RequestHandler } from './$types';
 import { rconStatusResult } from '$lib/server/integrations/rcon.js';
 import { getServer, setServerField } from '$lib/server/fleet/servers.js';
+import { latestGameSubmission } from '$lib/server/fleet/status-evidence.js';
+import { isProvisioning } from '$lib/server/fleet/provisioning.js';
 
 /**
  * On-demand RCON probe for a specific server. Same code path as the periodic
@@ -18,7 +20,14 @@ export const POST: RequestHandler = async ({ params }) => {
     const row = getServer(name);
     if (!row) return json({ ok: false, error: `unknown server: ${name}` }, { status: 404 });
 
+    const epoch = latestGameSubmission(name);
     const result = await rconStatusResult(name);
+    // Keep the diagnostic response, but never persist old-process evidence
+    // across a lifecycle request or replacement registration.
+    const current = getServer(name);
+    if (!current || current.created_at !== row.created_at || epoch !== latestGameSubmission(name) || isProvisioning(name)) {
+        return json(result);
+    }
     const now = new Date().toISOString();
 
     if (result.ok) {