Fix transient health checks and avoid unnecessary recovery

README.md

@@ -14,6 +14,14 @@ or restarting containers hide old game data and RCON errors; stopped, unhealthy,
 or unverifiable containers do not trigger routine RCON polling. Containers without
 a Docker health check use their running state as the readiness signal.
 
+Status polling runs every 30 seconds with at most four server checks in flight.
+Each completed check publishes a current fleet snapshot; a slow check does not
+hold back updates or the next poll for servers that have finished. RCON status
+queries retry timeouts and send errors twice, with 250ms then 500ms backoff.
+Overlapping status and occupancy probes share the in-flight query for the same
+server process and endpoint. Configuration and invalid-reply errors are reported
+immediately; commands that change server state are never retried by this logic.
+
 With fresh container health, validated telemetry received in the last 45 seconds
 supplies the current player count, roster, map, and game status. RCON is the fallback
 when telemetry is unavailable or stale. Fleet, server details, metrics, collection

docs/maintenance.md

@@ -106,10 +106,18 @@ The health observer (the watchdog) scans once per minute. Three observations ove
 at least two minutes are required before an automatic health restart; starting or
 restarting containers get five minutes. Unknown Komodo responses are observations,
 not proof of a stopped container. Repeated manual checks do not accelerate this
-window.
+window. A successful routine or manual game-status probe between scans resets
+the failure streak. Missing RCON configuration, unresolved ports, and invalid
+replies remain diagnostic findings and cannot authorize health recovery.
 
 A health finding gets one restart, then one rebuild if recovery fails after five
-minutes. Escalation transfers only that finding's restart slot. A failed rebuild
+minutes and fresh health checks still justify recovery. If current container and
+game checks succeed before an automatic health restart or rebuild is submitted,
+the obsolete operation is cancelled and the finding is resolved. Unknown health
+holds recovery for another check. Drift repairs and requested lifecycle operations
+retain their own eligibility rules.
+
+Escalation transfers only that finding's restart slot. A failed rebuild
 requires operator attention. **Rebuild deletes all instance files** and uses the
 latest promoted setup, the same server identity and port, and the saved server key.
 Missing keys block teardown. Submitted rebuild verification continues while paused.

src/lib/server/fleet/lifecycle-evidence.ts

@@ -0,0 +1,8 @@
+import { db } from '../db/index.js';
+
+/** Reports from before the latest lifecycle submission describe the old process. */
+export function latestGameSubmission(server: string): string | null {
+    return (db().prepare(`SELECT MAX(at) AS at FROM (
+        SELECT submitted_at AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
+        UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
+}

src/lib/server/fleet/status-evidence.ts

@@ -2,16 +2,12 @@ import { db } from '../db/index.js';
 import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
 import { ONLINE_MAX_MS } from '../../server-freshness.js';
 import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
+import { latestGameSubmission } from './lifecycle-evidence.js';
+
+export { latestGameSubmission } from './lifecycle-evidence.js';
 
 export interface TelemetryEvidence { players: number; map: string; receivedAt: string }
 
-/** Reports from before the latest lifecycle submission describe the old process. */
-export function latestGameSubmission(server: string): string | null {
-    return (db().prepare(`SELECT MAX(at) AS at FROM (
-        SELECT submitted_at AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
-        UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
-}
-
 export function telemetryReport(server: string): GameTelemetry | null {
     const row = db().prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?').get(server) as
         { received_at: string; payload: string | null } | undefined;

src/lib/server/integrations/komodo.ts

@@ -320,7 +320,7 @@ const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']);
 async function fetchJsonWithSnippet(
     path: string,
     body: Record<string, string>
-): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string }> {
+): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string; unsupportedEndpoint?: boolean }> {
     const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
     let res: Response;
     try {
@@ -330,7 +330,11 @@ async function fetchJsonWithSnippet(
     }
     const text = await res.text().catch(() => '');
     if (!res.ok) {
-        return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text) };
+        const unsupportedEndpoint = res.status === 404 || (
+            [400, 405, 422, 500, 501].includes(res.status)
+            && /\b(?:unknown|unsupported|unrecognized)\s+(?:method|endpoint|variant)\b|\bmethod not found\b/i.test(text)
+        );
+        return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text), unsupportedEndpoint };
     }
     let json: unknown;
     try {
@@ -382,9 +386,10 @@ export async function komodoInspectHealthy(serverName: string): Promise<KomodoHe
     for (const { path, body } of attempts) {
         const r = await fetchJsonWithSnippet(path, body);
         if (!r.ok) {
+            // Alternate endpoints handle API-version differences. Retrying an
+            // outage against every endpoint delays all other server probes.
+            if (!r.unsupportedEndpoint) return { ok: false, reason: r.reason, snippet: r.snippet };
             lastSnippet = r.snippet ?? lastSnippet;
-            // Try the next endpoint on 404s or unknown-method-shaped errors; a
-            // real error (auth, timeout) will fail every attempt anyway.
             continue;
         }
         const state = extractContainerState(r.json);

src/lib/server/integrations/rcon.ts

@@ -2,6 +2,7 @@ import { createSocket } from 'node:dgram';
 import type { Socket } from 'node:dgram';
 import { config, komodoConfigured, rconConfigured } from '../config.js';
 import { getServer, setServerField } from '../fleet/servers.js';
+import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
 import { komodoResolveGamePort } from './komodo.js';
 import { parsePlutoniumRevision } from '../../plutonium.js';
 
@@ -300,13 +301,8 @@ export function parsePlayerRoster(text: string): RconPlayer[] {
 }
 
 export async function rconGetPlayerCount(server: string): Promise<number | null> {
-    const res = await endpointFor(server);
+    const result = await rconStatusResult(server);
-    if (!res.endpoint) return null;
+    return result.ok ? result.status.players : null;
-    const send = await sendRcon(res.endpoint.host, res.endpoint.port, 'status');
-    if (!send.ok) return null;
-    const text = stripPrefix(send.buf);
-    if (!text) return null;
-    return parsePlayerCount(text);
 }
 
 export interface RconStatus {
@@ -338,6 +334,18 @@ export type RconStatusResult =
       };
 
 const RAW_REPLY_LIMIT = 512;
+const statusInFlight = new Map<string, Promise<RconStatusResult>>();
+
+/** Retry only this read-only query: commands with effects must never be replayed. */
+async function sendStatus(endpoint: RconEndpoint): Promise<SendResult> {
+    for (let attempt = 0; ; attempt++) {
+        const result = await sendRcon(endpoint.host, endpoint.port, 'status');
+        if (result.ok || (result.reason !== 'timeout' && result.reason !== 'send-error') || attempt === 2) {
+            return result;
+        }
+        await new Promise<void>(resolve => setTimeout(resolve, 250 * 2 ** attempt));
+    }
+}
 
 /**
  * Render a Buffer as a short hex + printable-ASCII snippet — same layout as
@@ -365,6 +373,10 @@ export function formatRawReply(buf: Buffer): string {
  * rather than just "unreachable".
  */
 export async function rconStatusResult(server: string): Promise<RconStatusResult> {
+    // Snapshot before any network work. A probe begun after a restart or a
+    // replacement registration must not join one observing the old process.
+    const createdAt = getServer(server)?.created_at;
+    const epoch = createdAt ? latestGameSubmission(server) : null;
     const res = await endpointFor(server);
     if (!res.endpoint) {
         if (res.reason === 'unknown-server') {
@@ -387,13 +399,22 @@ export async function rconStatusResult(server: string): Promise<RconStatusResult
     }
 
     const ep = res.endpoint;
+    const key = JSON.stringify([server, createdAt, epoch, ep.host, ep.port]);
+    const pending = statusInFlight.get(key);
+    if (pending) return pending;
+    const request = probeStatus(ep).finally(() => { statusInFlight.delete(key); });
+    statusInFlight.set(key, request);
+    return request;
+}
+
+async function probeStatus(ep: RconEndpoint): Promise<RconStatusResult> {
     // Plutonium T5 doesn't recognize `getstatus` as an rcon subcommand
     // ("Unknown command"), so we use `status` — a print-body response with
     // a header table plus one row per player. Q3-derived engines that
     // *do* answer statusResponse (via anonymous `getstatus` query) would
     // give richer info-line dvars, but every engine we care about supports
     // `status` under the rcon wrapper.
-    const send = await sendRcon(ep.host, ep.port, 'status');
+    const send = await sendStatus(ep);
     if (!send.ok) {
         return { ok: false, endpoint: ep, reason: send.reason, detail: send.detail };
     }

src/lib/server/maintenance/recovery.ts

@@ -5,9 +5,11 @@ import { getServer, listServers } from '../fleet/servers.js';
 import { hasActiveProvisioning } from '../fleet/provisioning.js';
 import { komodoInspectHealthy } from '../integrations/komodo.js';
 import { rconStatusResult } from '../integrations/rcon.js';
-import { telemetryEvidence } from '../fleet/status-evidence.js';
+import { latestGameSubmission, telemetryEvidence, telemetryReport } from '../fleet/status-evidence.js';
 import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '../fleet/drift.js';
-import { requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
+import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
+import { deploymentStage, finishDeployment } from '../provisioning/history.js';
+import { recordProgress } from '../provisioning/progress.js';
 import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident, WatchdogOverview, WatchdogState } from '../../watchdog.js';
 
 import { maintenanceAllowed, setMaintenancePolicy } from './policies.js';
@@ -43,7 +45,13 @@ function detect(server: string, kind: 'health' | 'drift', summary: string, detai
         const newSample = Date.now() - Date.parse(previous.last_seen_at) >= WATCHDOG_SCAN_MS;
         const actionable = (details as WatchdogEvidence & { actionable?: boolean }).actionable;
         const priorActionable = (JSON.parse(previous.details) as { actionable?: boolean }).actionable;
-        const reset = kind === 'health' && actionable === true && priorActionable !== true;
+        // A successful routine/manual probe between watchdog samples breaks
+        // the failure streak, even if our next probe happens to fail again.
+        const after = latestGameSubmission(server);
+        const successful = [getServer(server)?.last_seen_at, telemetryReport(server)?.receivedAt]
+            .some(at => at && Date.parse(at) > Date.parse(previous.last_seen_at)
+                && Date.parse(at) <= Date.now() && (!after || Date.parse(at) > Date.parse(after)));
+        const reset = kind === 'health' && (successful || (actionable === true && priorActionable !== true));
         db().prepare('UPDATE watchdog_incidents SET details=?, last_seen_at=?, failures=?, first_seen_at=? WHERE id=?')
             .run(JSON.stringify(details), newSample || reset ? iso() : previous.last_seen_at,
                 reset ? 1 : previous.failures + (newSample ? 1 : 0), reset ? iso() : previous.first_seen_at, previous.id);
@@ -56,6 +64,43 @@ function detect(server: string, kind: 'health' | 'drift', summary: string, detai
     event(id, summary);
     return incident(id)!;
 }
+
+/** Call only after verifying container readiness and current game evidence. */
+export function resolveRecoveredHealthOperation(operationId: string, players: number, source: 'rcon' | 'telemetry' | null): boolean {
+    return db().transaction(() => {
+        const operation = db().prepare(`SELECT d.server, d.action, d.state, d.submitted_at, d.triggered_by, i.id AS incident_id
+            FROM server_deployments d JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
+            JOIN watchdog_incidents i ON i.id = s.source_id AND i.server = d.server
+            WHERE d.id = ? AND d.server_deleted_at IS NULL AND i.server_deleted_at IS NULL
+                AND i.kind = 'health' AND i.state IN ('open','recovering')
+                AND NOT EXISTS (SELECT 1 FROM provisioning_sources other WHERE other.operation_id = d.id AND other.kind != 'watchdog')`)
+            .get(operationId) as { server: string; action: string; state: string; submitted_at: string | null; triggered_by: string; incident_id: string } | undefined;
+        if (!operation || getServer(operation.server)?.desired_state !== 'running') return false;
+        const waiting = operation.state === 'waiting' && operation.submitted_at === null;
+        if (waiting && !operation.triggered_by.startsWith('watchdog:')) return false;
+        // Cancelling our unsubmitted request is safe when later queued work
+        // exists, and lets that work proceed. Submitted recovery must retain
+        // exclusive ownership before accepting evidence or releasing its slot.
+        if (!waiting && hasActiveProvisioning(operation.server, operationId)) return false;
+        if (!waiting && (operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state))) return false;
+        const message = waiting ? 'Server recovered before automatic recovery was submitted' : 'Fresh container and game health verified after restart';
+        if (waiting && !cancelProvisioning(operationId, message)) return false;
+        const now = iso();
+        db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = NULL
+            WHERE server = ? AND operation_id = ? AND healthy_at IS NULL`).run(now, now, players, operation.server, operationId);
+        db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, operationId);
+        recordProgress(operationId, { blocker: null, observedAt: now,
+            recovery: { state: 'released', at: now, operationId, message },
+            checks: [{ key: 'container', label: 'Container readiness', source: 'komodo', result: 'pass', checkedAt: now },
+                { key: 'game', label: 'Game status', source, result: 'pass', actual: players, checkedAt: now }] });
+        if (operation.state === 'awaiting-health') {
+            deploymentStage(operationId, message);
+            finishDeployment(operationId, { ok: true });
+        }
+        transition(operation.incident_id, 'resolved', message);
+        return true;
+    })();
+}
 function actions(id: string): WatchdogAction[] {
     return db().prepare(`SELECT a.*, d.state AS operation_state, d.stage AS operation_stage, d.error AS operation_error
         FROM watchdog_actions a LEFT JOIN server_deployments d ON d.id=a.operation_id
@@ -157,6 +202,15 @@ async function reconcileActions(): Promise<void> {
         if (!maintenanceAllowed(row.kind)) continue;
         const current = getServer(row.server);
         if (!current || current.desired_state === 'stopped' || protectedWork(row.server)) continue;
+        // A recovery deadline is a reason to check again, not evidence that
+        // rebuilding is necessary. The worker may not have polled this tick yet.
+        const observation = await observe(row.server, action.operation_id);
+        if (!observation || incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue;
+        if (observation.healthy) {
+            resolveRecoveredHealthOperation(action.operation_id, observation.players!, observation.source ?? null);
+            continue;
+        }
+        if (!observation.actionable) continue;
         const result = db().transaction(() => {
             const next = escalateWatchdogRestart(row.id, action.operation_id, WATCHDOG_RECOVERY_MS);
             if (next.ok) addAction(row, next.deploymentId, 'rebuild');
@@ -166,16 +220,24 @@ async function reconcileActions(): Promise<void> {
     }
 }
 
-interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null }
+interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; players?: number; source?: 'rcon' | 'telemetry' }
-async function observe(server: string): Promise<Observation | null> {
+async function observe(server: string, operationId?: string): Promise<Observation | null> {
+    const epoch = latestGameSubmission(server);
+    const suppressed = () => (operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server))
+        || protectedWork(server) || getServer(server)?.desired_state !== 'running' || epoch !== latestGameSubmission(server);
     const health = await komodoInspectHealthy(server);
-    if (scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return null;
+    if (suppressed()) return null;
     const state = health.state?.toLowerCase() ?? null;
     if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
-    if (telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state };
+    const telemetry = telemetryEvidence(server);
+    if (telemetry) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
     const rcon = await rconStatusResult(server);
-    if (rcon.ok || telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state };
+    if (suppressed()) return null;
-    return { healthy: false, actionable: true, reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
+    const latest = telemetryEvidence(server);
+    if (rcon.ok || latest) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state,
+        players: latest?.players ?? (rcon.ok ? rcon.status.players : 0), source: latest ? 'telemetry' : 'rcon' };
+    return { healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
+        reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
 }
 async function scanServer(server: string): Promise<void> {
     const row = getServer(server);

src/lib/server/pipeline/worker.ts

@@ -6,7 +6,7 @@ import { emit } from './bus.js';
 import type { ServerStatusEntry } from './bus.js';
 import { rconStatusResult } from '../integrations/rcon.js';
 import type { RconFailure } from '../integrations/rcon.js';
-import { getServer, listServers, setServerField } from '../fleet/servers.js';
+import { getServer, listServers, setServerField, type ServerRow } from '../fleet/servers.js';
 import { komodoInspectHealthy } from '../integrations/komodo.js';
 import { restartSettling } from '../provisioning/index.js';
 import { serverObservation } from '../fleet/status.js';
@@ -58,92 +58,126 @@ function encodeError(reason: RconFailure, detail: string): string {
     return `${reason}:${detail}`;
 }
 
-let statusInFlight: Promise<void> | null = null;
+// Deduplicate queued/running checks per server. A slow server must not prevent
+// completed servers from being polled again on the next 30-second tick.
+const pendingStatus = new Map<string, Promise<number | null>>();
+const statusQueue: Array<() => Promise<void>> = [];
+let activeStatus = 0;
 
-export function fanoutStatus(): Promise<void> {
+function drainStatusQueue(): void {
-    if (statusInFlight) return statusInFlight;
+    while (activeStatus < STATUS_CONCURRENCY && statusQueue.length) {
-    statusInFlight = refreshStatus().finally(() => { statusInFlight = null; });
+        const next = statusQueue.shift()!;
-    return statusInFlight;
+        activeStatus++;
+        void next().finally(() => { activeStatus--; drainStatusQueue(); });
+    }
 }
 
-async function refreshStatus(): Promise<void> {
+function queueStatus(server: string): Promise<number | null> {
+    const pending = pendingStatus.get(server);
+    if (pending) return pending;
+    let finish!: (capacity: number | null) => void;
+    const work = new Promise<number | null>(resolve => { finish = resolve; });
+    pendingStatus.set(server, work);
+    statusQueue.push(async () => {
+        let capacity: number | null = null;
+        try {
+            capacity = await refreshServerStatus(server);
+            // Consumers expect a full snapshot. Re-read it now so neither game
+            // data nor container evidence waits for the slowest fleet member.
+            emit({ type: 'server-status', entries: listServers().map(statusEntry) });
+        } catch (err) { console.error(`[worker] status error for ${server}:`, err); }
+        finally { pendingStatus.delete(server); finish(capacity); }
+    });
+    drainStatusQueue();
+    return work;
+}
+
+export async function fanoutStatus(): Promise<void> {
     const servers = listServers();
     if (!servers.length) return;
-    const entries: ServerStatusEntry[] = new Array(servers.length);
+    const results = await Promise.all(servers.map(server => queueStatus(server.name)));
-    const capacities = new Map<string, number | null>();
+    const capacities = new Map(servers.map((server, i) => [server.name, results[i]]));
-    let cursor = 0;
+    try {
-    async function pump() {
+        // Metrics remain complete fleet collections, using freshness at capture
-        while (cursor < servers.length) {
+        // time rather than the status saved at the beginning of a long scan.
-            const i = cursor++;
+        recordMetrics(listServers().map(srv => {
-            const srv = servers[i];
+            const current = serverObservation(srv);
-            if (!isProvisioning(srv.name)) {
+            return { server: srv.name, game: srv.game ?? 'unknown',
-                const epoch = latestGameSubmission(srv.name);
+                map: selectedZombieMap(srv.game, current.last_map ?? srv.map, srv.map_slug)?.friendly ?? current.last_map ?? srv.map ?? 'Unknown map',
-                const container = restartSettling(srv.name)
+                players: current.health === 'online' ? current.last_players : null,
-                    ? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
+                capacity: capacities.get(srv.name) ?? null };
-                    : await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message }));
+        }));
-                // A lifecycle request may be submitted while an inspection is in flight.
+    } catch (err) { console.error('[worker] metrics error:', err); }
-                if (epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)) {
+}
-                    setServerField(srv.name, 'container_state', container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown');
+
-                    setServerField(srv.name, 'container_checked_at', new Date().toISOString());
+async function refreshServerStatus(server: string): Promise<number | null> {
-                    if (!container.ok) {
+    const srv = getServer(server);
-                        setServerField(srv.name, 'last_error', null);
+    if (!srv) return null;
-                        setServerField(srv.name, 'last_reply_snippet', null);
+    let capacity: number | null = null;
-                    } else if (!telemetryEvidence(srv.name)) {
+    if (!isProvisioning(srv.name)) {
-                        const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
+        const epoch = latestGameSubmission(srv.name);
-                            reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));
+        const unchanged = () => epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)
-                        if (epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)) {
+            && getServer(srv.name)?.created_at === srv.created_at;
-                            const now = new Date().toISOString();
+        const container = restartSettling(srv.name)
-                            setServerField(srv.name, 'last_checked_at', now);
+            ? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
-                            setServerField(srv.name, 'last_reply_snippet', result.rawReply ?? null);
+            : await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message }));
-                            if (result.ok) {
+        // A lifecycle request may be submitted while an inspection is in flight.
-                                capacities.set(srv.name, result.status.capacity ?? null);
+        if (unchanged()) {
-                                setServerField(srv.name, 'last_players', result.status.players);
+            setServerField(srv.name, 'container_state', container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown');
-                                setServerField(srv.name, 'last_map', result.status.map);
+            setServerField(srv.name, 'container_checked_at', new Date().toISOString());
-                                setServerField(srv.name, 'last_gametype', result.status.gametype);
+            if (!container.ok) {
-                                setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster));
-                                setServerField(srv.name, 'last_seen_at', now);
-                                setServerField(srv.name, 'last_error', null);
-                            } else setServerField(srv.name, 'last_error', encodeError(result.reason, result.detail));
-                        }
-                    }
-                }
-            } else {
                 setServerField(srv.name, 'last_error', null);
                 setServerField(srv.name, 'last_reply_snippet', null);
-            }
+            } else if (!telemetryEvidence(srv.name)) {
-            const current = getServer(srv.name) ?? srv;
+                const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
-            const status = serverObservation(current);
+                    reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));
-            // Occupancy timers and match clearing follow the same source as the UI.
+                if (unchanged()) {
-            if (status.health === 'online' && status.last_players !== null) {
+                    const now = new Date().toISOString();
-                if (status.last_players > 0) {
+                    setServerField(srv.name, 'last_checked_at', now);
-                    setServerField(srv.name, 'active_since', current.active_since ?? status.last_seen_at);
+                    setServerField(srv.name, 'last_reply_snippet', result.rawReply ?? null);
-                    setServerField(srv.name, 'inactive_since', null);
+                    if (result.ok) {
-                } else {
+                        capacity = result.status.capacity ?? null;
-                    setServerField(srv.name, 'active_since', null);
+                        setServerField(srv.name, 'last_players', result.status.players);
-                    setServerField(srv.name, 'inactive_since', current.inactive_since ?? status.last_seen_at);
+                        setServerField(srv.name, 'last_map', result.status.map);
-                    setServerField(srv.name, 'game_started_at', null);
+                        setServerField(srv.name, 'last_gametype', result.status.gametype);
-                    setServerField(srv.name, 'game_ended_at', null);
+                        setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster));
+                        setServerField(srv.name, 'last_seen_at', now);
+                        setServerField(srv.name, 'last_error', null);
+                    } else setServerField(srv.name, 'last_error', encodeError(result.reason, result.detail));
                 }
             }
-            const final = serverObservation(getServer(srv.name) ?? current);
+        }
-            entries[i] = { server: srv.name, players: final.last_players, map: final.last_map ?? srv.map,
+    } else {
-                gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at,
+        setServerField(srv.name, 'last_error', null);
-                lastError: final.last_error, replySnippet: final.replySnippet,
+        setServerField(srv.name, 'last_reply_snippet', null);
-                endpoint: current.game_port ? { host: current.rcon_host ?? config.rcon.host, port: current.game_port } : null, reason: final.last_error ? final.last_error.split(':')[0] as RconFailure : null,
+    }
-                activeSince: final.active_since, inactiveSince: final.inactive_since,
+    const current = getServer(srv.name) ?? srv;
-                gameStartedAt: final.game_started_at, gameEndedAt: final.game_ended_at,
+    const status = serverObservation(current);
-                containerState: final.container_state, containerCheckedAt: final.container_checked_at,
+    // Occupancy timers and match clearing follow the same source as the UI.
-                telemetryAfter: final.telemetry_after, rconStatus: final.rcon_status, telemetry: final.telemetry };
+    if (status.health === 'online' && status.last_players !== null) {
+        if (status.last_players > 0) {
+            setServerField(srv.name, 'active_since', current.active_since ?? status.last_seen_at);
+            setServerField(srv.name, 'inactive_since', null);
+        } else {
+            setServerField(srv.name, 'active_since', null);
+            setServerField(srv.name, 'inactive_since', current.inactive_since ?? status.last_seen_at);
+            setServerField(srv.name, 'game_started_at', null);
+            setServerField(srv.name, 'game_ended_at', null);
         }
     }
-    await Promise.all(Array.from({ length: Math.min(STATUS_CONCURRENCY, servers.length) }, pump));
+    return capacity;
-    try {
+}
-        recordMetrics(servers.map((srv, i) => ({ server: srv.name, game: srv.game ?? 'unknown',
+
-            map: selectedZombieMap(srv.game, entries[i].map ?? srv.map, srv.map_slug)?.friendly ?? entries[i].map ?? srv.map ?? 'Unknown map',
+function statusEntry(srv: ServerRow): ServerStatusEntry {
-            players: entries[i].health === 'online' ? entries[i].players : null,
+    const final = serverObservation(srv);
-            capacity: capacities.get(srv.name) ?? null })));
+    return { server: srv.name, players: final.last_players, map: final.last_map ?? srv.map,
-    } catch (err) { console.error('[worker] metrics error:', err); }
+        gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at,
-    emit({ type: 'server-status', entries });
+        lastError: final.last_error, replySnippet: final.replySnippet,
+        endpoint: srv.game_port ? { host: srv.rcon_host ?? config.rcon.host, port: srv.game_port } : null, reason: final.last_error ? final.last_error.split(':')[0] as RconFailure : null,
+        activeSince: final.active_since, inactiveSince: final.inactive_since,
+        gameStartedAt: final.game_started_at, gameEndedAt: final.game_ended_at,
+        containerState: final.container_state, containerCheckedAt: final.container_checked_at,
+        telemetryAfter: final.telemetry_after, rconStatus: final.rcon_status, telemetry: final.telemetry };
 }
 
 export function startWorker(): void {

src/lib/server/provisioning/requests.ts

@@ -139,12 +139,21 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
     return db().transaction(() => {
         const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;
         if (blocker) return { ok: false as const, status: 409, error: blocker };
-        const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
+        const existing = db().prepare(`SELECT id, action, state, submitted_at, triggered_by FROM server_deployments WHERE server = ?
             AND server_deleted_at IS NULL
-            AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
+            AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as
+            { id: string; action: string; state: string; submitted_at: string | null; triggered_by: string } | undefined;
         if (existing) {
             if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
             db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
+            if (existing.state === 'waiting' && !existing.submitted_at && existing.triggered_by.startsWith('watchdog:')
+                && db().prepare(`SELECT 1 FROM provisioning_sources s JOIN watchdog_incidents i ON i.id = s.source_id
+                    WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health'`).get(existing.id)) {
+                // An operator explicitly requesting this queued action keeps
+                // that intent even if the original health condition clears.
+                db().prepare("UPDATE server_deployments SET triggered_by = 'ui' WHERE id = ?").run(existing.id);
+                recordDeploymentEvent(existing.id, 'Operator explicitly requested the queued health recovery action');
+            }
             return { ok: true as const, deploymentId: existing.id };
         }
         const row = getServer(server);

src/lib/server/provisioning/worker.ts

@@ -1,11 +1,12 @@
 import { maintenanceAllowed } from '../maintenance/policies.js';
+import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
 import { db } from '../db/index.js';
 import { komodoConfigured, rconConfigured } from '../config.js';
 import { getServer } from '../fleet/servers.js';
 import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
 import { processCreations, processDestructions, rebuildServer } from './instances.js';
 import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
-import { probePlayerEvidence } from '../fleet/status-evidence.js';
+import { latestGameSubmission, probePlayerEvidence } from '../fleet/status-evidence.js';
 import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js';
 import { cancelProvisioning } from './requests.js';
 import type { RuntimeAction } from '../../provisioning.js';
@@ -158,11 +159,28 @@ async function processOperation(operation: Operation): Promise<void> {
     }
     const reason = blocker(server, id, action);
     if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; }
+    const healthRecovery = watchdog && !!db().prepare(`SELECT 1 FROM provisioning_sources s
+        JOIN watchdog_incidents i ON i.id = s.source_id JOIN server_deployments d ON d.id = s.operation_id
+        WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health' AND d.triggered_by LIKE 'watchdog:%'`).get(id);
+    const epoch = latestGameSubmission(server);
     const health = await komodoInspectHealthy(server);
+    if (healthRecovery && epoch !== latestGameSubmission(server)) return;
+    if (healthRecovery && !health.ok && !['stopped','exited','created','dead','unhealthy','starting','restarting'].includes(health.state?.toLowerCase() ?? '')) {
+        deploymentStage(id, 'Waiting for current container health before automatic recovery', { phase: 'eligibility', checks: [],
+            blocker: { code: 'observation-unavailable', message: health.reason, automatic: true } });
+        return;
+    }
     if (action !== 'start' && !isStopped(health.state)) {
         const evidence = await probePlayerEvidence(server);
         if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
         const players = evidence.players;
+        if (healthRecovery && epoch !== latestGameSubmission(server)) return;
+        if (healthRecovery && health.ok && players !== null) {
+            // The fresh occupancy check also proves that the game recovered.
+            // Empty is not a reason to execute an obsolete health restart.
+            resolveRecoveredHealthOperation(id, players, evidence.source);
+            return;
+        }
         recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting',
             source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }],
             blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present',

src/routes/api/servers/[name]/probe-rcon/+server.ts

@@ -2,6 +2,8 @@ import { json } from '@sveltejs/kit';
 import type { RequestHandler } from './$types';
 import { rconStatusResult } from '$lib/server/integrations/rcon.js';
 import { getServer, setServerField } from '$lib/server/fleet/servers.js';
+import { latestGameSubmission } from '$lib/server/fleet/status-evidence.js';
+import { isProvisioning } from '$lib/server/fleet/provisioning.js';
 
 /**
  * On-demand RCON probe for a specific server. Same code path as the periodic
@@ -18,7 +20,14 @@ export const POST: RequestHandler = async ({ params }) => {
     const row = getServer(name);
     if (!row) return json({ ok: false, error: `unknown server: ${name}` }, { status: 404 });
 
+    const epoch = latestGameSubmission(name);
     const result = await rconStatusResult(name);
+    // Keep the diagnostic response, but never persist old-process evidence
+    // across a lifecycle request or replacement registration.
+    const current = getServer(name);
+    if (!current || current.created_at !== row.created_at || epoch !== latestGameSubmission(name) || isProvisioning(name)) {
+        return json(result);
+    }
     const now = new Date().toISOString();
 
     if (result.ok) {

tests/unit/komodo-health.test.ts

@@ -0,0 +1,106 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { komodoInspectHealthy } from '../../src/lib/server/integrations/komodo.js';
+
+vi.mock('../../src/lib/server/config.js', () => ({
+    config: {
+        komodo: { address: 'http://komodo.test', apiKey: 'test', apiSecret: 'test' },
+        httpTimeoutMs: 1000
+    },
+    komodoConfigured: () => true
+}));
+vi.mock('../../src/lib/server/fleet/servers.js', () => ({
+    getServer: () => ({ komodo_type: 'deployment', komodo_name: 'saved-resource' })
+}));
+
+const fetchMock = vi.fn<typeof fetch>();
+
+beforeEach(() => {
+    fetchMock.mockReset();
+    vi.stubGlobal('fetch', fetchMock);
+});
+afterEach(() => {
+    vi.unstubAllGlobals();
+    vi.useRealTimers();
+});
+
+describe('Komodo health endpoint fallback', () => {
+    it('limits an unreachable probe to one timeout and preserves its diagnosis', async () => {
+        vi.useFakeTimers();
+        fetchMock.mockImplementation((_url, options) => new Promise((_resolve, reject) => {
+            options?.signal?.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError')));
+        }));
+
+        const result = komodoInspectHealthy('server');
+        await vi.advanceTimersByTimeAsync(1000);
+
+        expect(fetchMock).toHaveBeenCalledTimes(1);
+        expect(await result).toMatchObject({
+            ok: false,
+            reason: 'timed out after 1000ms dialing http://komodo.test/read/GetDeploymentContainer'
+        });
+    });
+
+    it('preserves the network cause without probing alternate endpoints', async () => {
+        fetchMock.mockRejectedValue(new TypeError('fetch failed', {
+            cause: Object.assign(new Error('connection refused'), { code: 'ECONNREFUSED' })
+        }));
+
+        expect(await komodoInspectHealthy('server')).toMatchObject({
+            ok: false,
+            reason: expect.stringContaining('ECONNREFUSED: connection refused')
+        });
+        expect(fetchMock).toHaveBeenCalledTimes(1);
+    });
+
+    it.each([400, 401, 403, 429, 500, 502, 503])('stops on HTTP %i without hiding the response', async status => {
+        fetchMock.mockResolvedValue(new Response('request failed', { status }));
+
+        expect(await komodoInspectHealthy('server')).toEqual({
+            ok: false, reason: `HTTP ${status}`, snippet: 'request failed'
+        });
+        expect(fetchMock).toHaveBeenCalledTimes(1);
+    });
+
+    it.each([
+        [404, 'Not Found'],
+        [400, 'unknown variant `GetDeploymentContainer`'],
+        [405, 'unsupported method'],
+        [500, 'unknown method'],
+        [501, 'method not found']
+    ])('tries a compatible endpoint after HTTP %i: %s', async (status, message) => {
+        fetchMock
+            .mockResolvedValueOnce(new Response(message as string, { status: status as number }))
+            .mockResolvedValueOnce(Response.json({ State: { Status: 'running' } }));
+
+        expect(await komodoInspectHealthy('server')).toEqual({ ok: true, state: 'running' });
+        expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([
+            'http://komodo.test/read/GetDeploymentContainer',
+            'http://komodo.test/read/InspectDeploymentContainer'
+        ]);
+    });
+
+    it('continues past successful responses with an unknown shape', async () => {
+        fetchMock
+            .mockResolvedValueOnce(Response.json({ config: { image: 'server' } }))
+            .mockResolvedValueOnce(Response.json({ State: { Status: 'running' } }));
+
+        expect(await komodoInspectHealthy('server')).toEqual({ ok: true, state: 'running' });
+        expect(fetchMock).toHaveBeenCalledTimes(2);
+    });
+
+    it('does not overwrite an unhealthy container state with a later response', async () => {
+        fetchMock.mockResolvedValueOnce(Response.json({ State: { Status: 'exited' } }));
+
+        expect(await komodoInspectHealthy('server')).toMatchObject({ ok: false, state: 'exited', reason: 'container state: exited' });
+        expect(fetchMock).toHaveBeenCalledTimes(1);
+    });
+
+    it('preserves malformed response diagnostics', async () => {
+        fetchMock.mockResolvedValueOnce(new Response('not json'));
+
+        expect(await komodoInspectHealthy('server')).toMatchObject({
+            ok: false, reason: expect.stringContaining('unparseable json:'), snippet: 'not json'
+        });
+        expect(fetchMock).toHaveBeenCalledTimes(1);
+    });
+});

tests/unit/provisioning.test.ts

@@ -256,6 +256,10 @@ describe('provisioning operations', () => {
 
 async function incident(id = 'incident') {
     const { db } = await import('../../src/lib/server/db/index.js');
+    const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
+    // These lifecycle tests exercise an actual container failure. Healthy
+    // containers with fresh game evidence now cancel obsolete health work.
+    vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'unhealthy', reason: 'Container unhealthy' });
     const now = new Date().toISOString();
     db().prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at)
         VALUES (?,?,'health','open','Server down',?,?)`).run(id, server, now, now);
@@ -323,7 +327,7 @@ describe('Watchdog provisioning and telemetry evidence', () => {
         const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
         const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
         const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
-        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
+        const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
         const { config } = await import('../../src/lib/server/config.js');
         vi.spyOn(config.rcon, 'password', 'get').mockReturnValue('');
         vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
@@ -333,6 +337,7 @@ describe('Watchdog provisioning and telemetry evidence', () => {
         if (!result.ok) throw new Error(result.error);
         await processProvisioning();
         expect(komodoRestart).toHaveBeenCalledTimes(1);
+        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
         vi.setSystemTime(Date.now() + 15_000);
         await processProvisioning();
         expect(await operation(result.deploymentId)).toMatchObject({ state: 'awaiting-health' });

tests/unit/rcon-status.test.ts

@@ -0,0 +1,167 @@
+import { EventEmitter } from 'node:events';
+import { createSocket, type Socket } from 'node:dgram';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { komodoConfigured, rconConfigured } from '../../src/lib/server/config.js';
+import { getServer, type ServerRow } from '../../src/lib/server/fleet/servers.js';
+import { latestGameSubmission } from '../../src/lib/server/fleet/lifecycle-evidence.js';
+import { rconGetPlayerCount, rconPlutoniumVersion, rconStatusResult } from '../../src/lib/server/integrations/rcon.js';
+
+vi.mock('node:dgram', () => ({ createSocket: vi.fn() }));
+vi.mock('../../src/lib/server/config.js', () => ({
+    config: { rcon: { host: 'localhost', password: 'test', timeoutMs: 1_000 } },
+    komodoConfigured: vi.fn(), rconConfigured: vi.fn()
+}));
+vi.mock('../../src/lib/server/fleet/servers.js', () => ({ getServer: vi.fn(), setServerField: vi.fn() }));
+vi.mock('../../src/lib/server/fleet/lifecycle-evidence.js', () => ({ latestGameSubmission: vi.fn() }));
+vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoResolveGamePort: vi.fn() }));
+
+class FakeSocket extends EventEmitter {
+    send = vi.fn();
+    close = vi.fn();
+}
+
+let sockets: FakeSocket[];
+let server: ServerRow;
+const packet = (body: string) => Buffer.concat([Buffer.from([255, 255, 255, 255]), Buffer.from(body)]);
+const reply = (index = 0) => sockets[index].emit('message', packet('statusResponse\n\\mapname\\mp_kino\n10 45 "joe"'));
+
+beforeEach(() => {
+    vi.resetAllMocks();
+    vi.useFakeTimers();
+    sockets = [];
+    server = { name: 'test-server', created_at: '2026-10-01T00:00:00Z', game_port: 28960, rcon_host: null } as ServerRow;
+    vi.mocked(getServer).mockImplementation(() => ({ ...server }));
+    vi.mocked(latestGameSubmission).mockReturnValue(null);
+    vi.mocked(komodoConfigured).mockReturnValue(false);
+    vi.mocked(rconConfigured).mockReturnValue(true);
+    vi.mocked(createSocket).mockImplementation(() => {
+        const socket = new FakeSocket();
+        sockets.push(socket);
+        return socket as unknown as Socket;
+    });
+});
+
+afterEach(() => {
+    expect(vi.getTimerCount()).toBe(0);
+    vi.useRealTimers();
+});
+
+describe('RCON status transport', () => {
+    it('shares overlapping health and occupancy probes and probes again after completion', async () => {
+        const health = rconStatusResult(server.name);
+        const occupancy = rconGetPlayerCount(server.name);
+        const duplicate = rconStatusResult(server.name);
+        await vi.advanceTimersByTimeAsync(0);
+        expect(sockets).toHaveLength(1);
+        expect(sockets[0].send.mock.calls[0][0].toString()).toContain('rcon test status\n');
+        reply();
+        expect(await health).toMatchObject({ ok: true, status: { players: 1, map: 'mp_kino' } });
+        expect(await occupancy).toBe(1);
+        expect(await duplicate).toEqual(await health);
+        expect(sockets[0].close).toHaveBeenCalledOnce();
+
+        const fresh = rconStatusResult(server.name);
+        await vi.advanceTimersByTimeAsync(0);
+        expect(sockets).toHaveLength(2);
+        reply(1);
+        expect(await fresh).toMatchObject({ ok: true });
+    });
+
+    it('recovers a dropped UDP reply after the first backoff', async () => {
+        const request = rconStatusResult(server.name);
+        await vi.advanceTimersByTimeAsync(1_000);
+        expect(sockets).toHaveLength(1);
+        expect(sockets[0].close).toHaveBeenCalledOnce();
+        await vi.advanceTimersByTimeAsync(249);
+        expect(sockets).toHaveLength(1);
+        await vi.advanceTimersByTimeAsync(1);
+        expect(sockets).toHaveLength(2);
+        reply(1);
+        expect(await request).toMatchObject({ ok: true });
+    });
+
+    it('bounds retries at three attempts with 250ms then 500ms backoffs', async () => {
+        const request = rconStatusResult(server.name);
+        await vi.advanceTimersByTimeAsync(1_250);
+        expect(sockets).toHaveLength(2);
+        await vi.advanceTimersByTimeAsync(1_499);
+        expect(sockets).toHaveLength(2);
+        await vi.advanceTimersByTimeAsync(1);
+        expect(sockets).toHaveLength(3);
+        await vi.advanceTimersByTimeAsync(1_000);
+        expect(await request).toMatchObject({ ok: false, reason: 'timeout' });
+        expect(sockets.every(socket => socket.close.mock.calls.length === 1)).toBe(true);
+
+        const fresh = rconStatusResult(server.name);
+        await vi.advanceTimersByTimeAsync(0);
+        expect(sockets).toHaveLength(4);
+        reply(3);
+        expect(await fresh).toMatchObject({ ok: true });
+    });
+
+    it.each(['event', 'callback'])('retries transient send errors reported through the %s', async source => {
+        const request = rconStatusResult(server.name);
+        await vi.advanceTimersByTimeAsync(0);
+        if (source === 'event') sockets[0].emit('error', new Error('UDP unavailable'));
+        else sockets[0].send.mock.calls[0][5](new Error('UDP unavailable'));
+        await vi.advanceTimersByTimeAsync(250);
+        expect(sockets).toHaveLength(2);
+        reply(1);
+        expect(await request).toMatchObject({ ok: true });
+    });
+
+    it.each([
+        ['missing prefix', Buffer.from('wrong listener')],
+        ['wrong password', packet('print\nBad rcon_password')],
+        ['unknown dialect', packet('unexpected reply')]
+    ])('does not retry a %s reply', async (_name, response) => {
+        const request = rconStatusResult(server.name);
+        await vi.advanceTimersByTimeAsync(0);
+        sockets[0].emit('message', response);
+        expect(await request).toMatchObject({ ok: false, reason: 'bad-reply' });
+        expect(sockets).toHaveLength(1);
+    });
+
+    it('does not retry missing configuration or endpoint failures', async () => {
+        vi.mocked(rconConfigured).mockReturnValue(false);
+        expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason: 'no-config' });
+        server.game_port = null;
+        expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason: 'no-port' });
+        vi.mocked(getServer).mockReturnValue(undefined);
+        expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason: 'no-port' });
+        expect(sockets).toHaveLength(0);
+    });
+
+    it('cleans up a rejected in-flight probe so the next one can proceed', async () => {
+        vi.mocked(createSocket).mockImplementationOnce(() => { throw new Error('Socket initialization failed'); });
+        await expect(rconStatusResult(server.name)).rejects.toThrow('Socket initialization failed');
+        const request = rconStatusResult(server.name);
+        await vi.advanceTimersByTimeAsync(0);
+        reply();
+        expect(await request).toMatchObject({ ok: true });
+    });
+
+    it.each(['restart', 'registration', 'host', 'port', 'server'])('does not share probes across a changed %s', async change => {
+        const original = rconStatusResult(server.name);
+        await vi.advanceTimersByTimeAsync(0);
+        if (change === 'restart') vi.mocked(latestGameSubmission).mockReturnValue('2026-10-02T00:00:00Z');
+        if (change === 'registration') server.created_at = '2026-10-02T00:00:00Z';
+        if (change === 'host') server.rcon_host = 'new-host';
+        if (change === 'port') server.game_port = 28961;
+        if (change === 'server') server.name = 'other-server';
+        const current = rconStatusResult(server.name);
+        await vi.advanceTimersByTimeAsync(0);
+        expect(sockets).toHaveLength(2);
+        reply();
+        sockets[1].emit('message', packet('statusResponse\n\\mapname\\mp_ascension\n'));
+        expect(await original).toMatchObject({ ok: true, status: { players: 1 } });
+        expect(await current).toMatchObject({ ok: true, status: { players: 0 } });
+    });
+
+    it('leaves other RCON commands on their existing single-attempt behavior', async () => {
+        const request = rconPlutoniumVersion(server.name);
+        await vi.advanceTimersByTimeAsync(1_000);
+        expect(await request).toMatchObject({ revision: null, error: 'no reply within 1000ms' });
+        expect(sockets).toHaveLength(1);
+    });
+});

tests/unit/server-status.test.ts

@@ -30,6 +30,110 @@ async function setup() {
 }
 
 describe('database fleet and container readiness', () => {
+    it('refreshes a ten-server fleet while one check is slow, without duplicating or exceeding four checks', async () => {
+        const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
+        const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const now = Date.parse('2026-10-02T12:00:00Z');
+        vi.setSystemTime(now);
+        for (let i = 0; i < 9; i++) db.prepare('INSERT INTO servers (name, komodo_name) VALUES (?, ?)').run(`fast-${i}`, `fast-${i}`);
+        let release!: (result: { ok: true; state: string }) => void;
+        let active = 0;
+        let peak = 0;
+        inspect.mockImplementation(async name => {
+            peak = Math.max(peak, ++active);
+            try {
+                await Promise.resolve();
+                return name === 'managed'
+                    ? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; })
+                    : { ok: true as const, state: 'running' };
+            } finally { active--; }
+        });
+        rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
+            status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } });
+        const events: unknown[] = [];
+        const unsubscribe = subscribe(event => events.push(event));
+        const first = fanoutStatus();
+        try {
+            await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(9));
+            expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: expect.arrayContaining([
+                expect.objectContaining({ server: 'fast-8', health: 'online', players: 2 })
+            ]) }));
+            vi.setSystemTime(now + 30_000);
+            const second = fanoutStatus();
+            await vi.waitFor(() => expect(Date.parse(getServer('fast-8')?.last_seen_at ?? '')).toBeGreaterThanOrEqual(now + 30_000));
+            expect(rcon).toHaveBeenCalledTimes(18);
+            expect(inspect.mock.calls.filter(([name]) => name === 'managed')).toHaveLength(1);
+            expect(peak).toBe(4);
+            release({ ok: true, state: 'running' });
+            await Promise.all([first, second]);
+            expect(rcon).toHaveBeenCalledTimes(19);
+        } finally {
+            release({ ok: true, state: 'running' });
+            await first;
+            unsubscribe();
+        }
+    });
+
+    it('expires early results before recording a fleet collection delayed by a slow check', async () => {
+        const { db, inspect, rcon, fanoutStatus } = await setup();
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const now = Date.parse('2026-10-02T12:00:00Z');
+        vi.setSystemTime(now);
+        db.prepare("INSERT INTO servers (name, komodo_name) VALUES ('slow', 'slow')").run();
+        let release!: (result: { ok: true; state: string }) => void;
+        inspect.mockImplementation(async name => name === 'slow'
+            ? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; })
+            : { ok: true as const, state: 'running' });
+        rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
+            status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } });
+        const pending = fanoutStatus();
+        try {
+            await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1));
+            vi.setSystemTime(now + 46_000);
+        } finally {
+            release({ ok: true, state: 'running' });
+            await pending;
+        }
+        expect(db.prepare('SELECT server, players FROM metric_samples ORDER BY server').all()).toEqual([
+            { server: 'managed', players: null }, { server: 'slow', players: 2 }
+        ]);
+    });
+
+    it('keeps a manual probe diagnostic without persisting it across a restart', async () => {
+        const { db, rcon, getServer } = await setup();
+        const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
+        let release!: (result: Awaited<ReturnType<typeof rcon>>) => void;
+        rcon.mockImplementation(() => new Promise(resolve => { release = resolve; }));
+        const pending = POST({ params: { name: 'managed' } } as never);
+        db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString());
+        release({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old status',
+            status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
+        expect(await (await pending).json()).toMatchObject({ ok: true });
+        expect(getServer('managed')).toMatchObject({ last_seen_at: null, last_error: 'timeout:old process' });
+    });
+
+    it.each(['container', 'rcon'])('ignores an old registration whose %s check finishes after replacement', async phase => {
+        const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
+        let release!: () => void;
+        const blocked = new Promise<void>(resolve => { release = resolve; });
+        inspect.mockImplementation(async () => {
+            if (phase === 'container') await blocked;
+            return { ok: true, state: 'running' };
+        });
+        rcon.mockImplementation(async () => {
+            if (phase === 'rcon') await blocked;
+            return { ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old status',
+                status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } };
+        });
+        const pending = fanoutStatus();
+        try {
+            if (phase === 'rcon') await vi.waitFor(() => expect(rcon).toHaveBeenCalled());
+            db.prepare("UPDATE servers SET created_at = 'replacement', container_state = NULL, last_error = NULL WHERE name = 'managed'").run();
+        } finally { release(); await pending; }
+        expect(getServer('managed')).toMatchObject({ container_state: null, last_seen_at: null, last_error: null });
+    });
+
     it.each(['queued', 'running', 'awaiting-health'])('shows provisioning during %s creation and resumes health reporting afterward', async (state) => {
         const { db, inspect, rcon, fanoutStatus } = await setup();
         const { GET } = await import('../../src/routes/api/servers/+server.js');

tests/unit/watchdog.test.ts

@@ -153,6 +153,105 @@ describe('Watchdog detection and ownership', () => {
         expect(operations()).toEqual([]);
     });
 
+    it.each(['no-config', 'no-port', 'bad-reply', 'no-players'] as const)('keeps %s RCON failures observational', async reason => {
+        vi.mocked(rcon.rconStatusResult).mockResolvedValue({ ...missingRcon, reason });
+        for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
+        expect(onlyIncident()).toMatchObject({ state: 'open' });
+        expect(operations()).toEqual([]);
+    });
+
+    it('breaks the sustained failure window on a successful probe between watchdog samples', async () => {
+        vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
+        await watchdog.watchdogTick(true);
+        advance(watchdog.WATCHDOG_SCAN_MS);
+        await watchdog.watchdogTick(true);
+        advance(20_000);
+        const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
+        vi.mocked(rcon.rconStatusResult).mockResolvedValueOnce(healthyRcon);
+        await POST({ params: { name: server } } as Parameters<typeof POST>[0]);
+        advance(40_000);
+        await watchdog.watchdogTick(true);
+        expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open', first_seen_at: new Date().toISOString() });
+        expect(operations()).toEqual([]);
+        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
+        expect(operations()).toEqual([]);
+        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
+        expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
+    });
+
+    it.each([0, 2])('cancels queued health recovery when fresh RCON responds with %i players', async players => {
+        const row = await sustainedFailure();
+        vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
+        vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(players);
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' })]);
+        expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
+        expect(komodo.komodoRestart).not.toHaveBeenCalled();
+    });
+
+    it('holds queued health recovery when container readiness becomes unknown', async () => {
+        await sustainedFailure();
+        vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' });
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        expect(operations()[0].state).toBe('waiting');
+        expect(komodo.komodoRestart).not.toHaveBeenCalled();
+    });
+
+    it('does not apply a recovered occupancy probe after its operation is replaced', async () => {
+        const row = await sustainedFailure();
+        const restart = operations()[0];
+        const { cancelProvisioning, requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
+        vi.mocked(rcon.rconGetPlayerCount).mockImplementationOnce(async () => {
+            expect(cancelProvisioning(restart.id)).toBe(true);
+            expect(requestLifecycle(server, 'stop').ok).toBe(true);
+            return 0;
+        });
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
+        expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' }),
+            expect.objectContaining({ action: 'stop', state: 'waiting' })]);
+        expect(komodo.komodoRestart).not.toHaveBeenCalled();
+    });
+
+    it('cancels obsolete health recovery so a later queued maintenance restart can proceed', async () => {
+        const row = await sustainedFailure();
+        const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
+        const job = database.db().prepare('INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)')
+            .run(server, new Date().toISOString()).lastInsertRowid;
+        const maintenance = requestRestart(server, { kind: 'maintenance', id: String(job) });
+        vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
+        expect(operations()).toEqual([expect.objectContaining({ state: 'cancelled' }),
+            expect.objectContaining({ id: maintenance.id, state: 'awaiting-health' })]);
+        expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
+    });
+
+    it.each(['restart', 'rebuild'] as const)('preserves an operator request adopting a queued health %s', async action => {
+        await sustainedFailure();
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        if (action === 'rebuild') {
+            await processProvisioning();
+            advance(watchdog.WATCHDOG_RECOVERY_MS);
+            await watchdog.watchdogTick(true);
+        }
+        const queued = operations().at(-1)!;
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        expect(requestLifecycle(server, action)).toEqual({ ok: true, deploymentId: queued.id });
+        expect(database.db().prepare('SELECT triggered_by FROM server_deployments WHERE id = ?').get(queued.id)).toEqual({ triggered_by: 'ui' });
+        vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
+        if (action === 'rebuild') vi.mocked(komodo.komodoDestroyDeployment).mockImplementationOnce(async () => { down(); return { ok: true }; });
+        await processProvisioning();
+        expect(operations().at(-1)).toMatchObject({ id: queued.id, action, state: action === 'restart' ? 'awaiting-health' : 'queued' });
+        if (action === 'restart') expect(komodo.komodoRestart).toHaveBeenCalledTimes(1);
+        else expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1);
+    });
+
     it('preserves intentional stop intent after completion and database reopening', async () => {
         const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
         const stop = requestLifecycle(server, 'stop');
@@ -241,6 +340,79 @@ describe('Watchdog detection and ownership', () => {
 });
 
 describe('Watchdog recovery persistence', () => {
+    it('verifies recovery at the deadline before deciding whether to rebuild', async () => {
+        const row = await sustainedFailure();
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        advance(watchdog.WATCHDOG_RECOVERY_MS);
+        vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
+        await watchdog.watchdogTick(true);
+        expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'done' })]);
+        expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
+        expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server))
+            .toEqual({ healthy_at: new Date().toISOString() });
+    });
+
+    it('defers escalation while current container health is unknown', async () => {
+        await sustainedFailure();
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        advance(watchdog.WATCHDOG_RECOVERY_MS);
+        vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' });
+        await watchdog.watchdogTick(true);
+        expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'awaiting-health' })]);
+    });
+
+    it('keeps failed submission history when fresh evidence later releases recovery', async () => {
+        const row = await sustainedFailure();
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        vi.mocked(komodo.komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Reply lost' });
+        await processProvisioning();
+        advance(watchdog.WATCHDOG_RECOVERY_MS);
+        vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
+        await watchdog.watchdogTick(true);
+        expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'failed' })]);
+        expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
+        expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server))
+            .toEqual({ healthy_at: new Date().toISOString() });
+        expect(watchdog.watchdogDetail(row.id)?.actions[0].operation_error).toContain('Reply lost');
+    });
+
+    it('discards escalation evidence if another lifecycle submission appears during its probe', async () => {
+        const row = await sustainedFailure();
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        advance(watchdog.WATCHDOG_RECOVERY_MS);
+        vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
+        vi.mocked(rcon.rconStatusResult).mockImplementationOnce(async () => {
+            database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
+                VALUES ('newer', ?, 'restart', 'done', 'Complete', ?, ?, 'manual')`).run(server, new Date().toISOString(), new Date().toISOString());
+            return healthyRcon;
+        });
+        await watchdog.watchdogTick(true);
+        expect(operations()[0]).toMatchObject({ action: 'restart', state: 'awaiting-health' });
+        expect(operations().some(op => op.action === 'rebuild')).toBe(false);
+        expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering');
+        expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
+    });
+
+    it('cancels a queued health rebuild and releases its transferred slot when the game recovers', async () => {
+        const row = await sustainedFailure();
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        await processProvisioning();
+        advance(watchdog.WATCHDOG_RECOVERY_MS);
+        await watchdog.watchdogTick(true);
+        expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']);
+        const rebuild = operations()[1];
+        vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
+        await processProvisioning();
+        expect(operations()).toEqual([expect.objectContaining({ state: 'failed' }), expect.objectContaining({ state: 'cancelled' })]);
+        expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
+        expect(database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server))
+            .toEqual({ operation_id: rebuild.id, healthy_at: new Date().toISOString() });
+        expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled();
+    });
+
     it('preserves the original finding as deleted history after an intentional successful destroy', async () => {
         const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
         const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');