Use status monitor evidence for routine watchdog health checks

src/lib/server/fleet/health-observations.ts

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto';
 import { config } from '../config.js';
 import { db } from '../db/index.js';
 import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js';
@@ -7,6 +8,7 @@ import { rconStatusResult, agentRconStatusResult, type AgentRconReply } from '..
 import { withOperationSource } from '../hosts/operation-context.js';
 import { getServer } from './servers.js';
 import { latestGameSubmission } from './lifecycle-evidence.js';
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
 
 const MAX_AGE_MS = 30_000;
 const MAX_SERVERS = 1024;
@@ -100,6 +102,62 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
 const inspections = new Map<string, Entry<Awaited<ReturnType<typeof komodoInspectHealthy>>>>();
 const rcon = new Map<string, Entry<Awaited<ReturnType<typeof rconStatusResult>>>>();
 
+interface MonitoringStatus {
+    identity: string;
+    container: Observation<Awaited<ReturnType<typeof komodoInspectHealthy>>>;
+    game?: Observation<Awaited<ReturnType<typeof rconStatusResult>>>;
+}
+// Keep failed samples too, independently of the successful-probe cache. Reading
+// this snapshot must never dispatch a request or renew an observation's age.
+const statuses = new Map<string, { snapshot?: MonitoringStatus }>();
+
+export function monitoringIdentity(server: string): string {
+    const container = identity(server, 'inspect');
+    const game = identity(server, 'rcon');
+    // Identity includes connection configuration; persist only its fingerprint.
+    return createHash('sha256').update(JSON.stringify([container.key, game.key, game.port])).digest('hex');
+}
+
+/** Capture ownership before a status pass; publish only evidence for that instance. */
+export function monitoringStatusWriter(server: string) {
+    let before: string;
+    try { before = identity(server, 'inspect').key; }
+    catch { return () => {}; }
+    const slot = { snapshot: statuses.get(server)?.snapshot };
+    statuses.delete(server);
+    statuses.set(server, slot);
+    while (statuses.size > MAX_SERVERS) statuses.delete(statuses.keys().next().value!);
+    return (container: MonitoringStatus['container'], game?: MonitoringStatus['game']) => {
+        // A forced check or newer pass supersedes this writer, even while its
+        // network requests are still in flight.
+        if (statuses.get(server) !== slot) return;
+        try {
+            if (identity(server, 'inspect').key !== before) return;
+            const current = slot.snapshot;
+            if (current && current.container.observedAt > container.observedAt) return;
+            slot.snapshot = { identity: monitoringIdentity(server), container, game };
+        } catch { statuses.delete(server); }
+    };
+}
+
+/** Latest completed status-monitor pass, with stale or superseded evidence excluded. */
+export function monitoringStatus(server: string): MonitoringStatus | null {
+    const saved = statuses.get(server)?.snapshot;
+    if (!saved) return null;
+    const fresh = (at: string) => {
+        const age = Date.now() - Date.parse(at);
+        return age >= 0 && age <= ONLINE_MAX_MS;
+    };
+    try {
+        if (saved.identity !== monitoringIdentity(server)
+            || !fresh(saved.container.observedAt)
+            || newerEvidence(identity(server, 'inspect'), Date.parse(saved.container.observedAt))) return null;
+        const game = saved.game && fresh(saved.game.observedAt)
+            && !newerEvidence(identity(server, 'rcon'), Date.parse(saved.game.observedAt)) ? saved.game : undefined;
+        return { ...saved, game };
+    } catch { return null; }
+}
+
 /** Keep the actual sample time: a later heartbeat must not renew old evidence. */
 export function heartbeatInspection(server: string): Observation<Awaited<ReturnType<typeof komodoInspectHealthy>>> | null {
     const row = getServer(server);
@@ -175,10 +233,12 @@ export function prepareMonitoringRcon(servers: string[]): Map<string, () => Prom
 
 /** Routine monitoring only. Manual diagnostics and disruption gates use fresh probes directly. */
 export function inspectForMonitoring(server: string, fresh = false) {
+    if (fresh) statuses.delete(server);
     const heartbeat = !fresh && heartbeatInspection(server);
     if (heartbeat) return Promise.resolve(heartbeat);
     return observe(server, fresh, 'inspect', inspections, komodoInspectHealthy);
 }
 export function rconForMonitoring(server: string, fresh = false) {
+    if (fresh) statuses.delete(server);
     return observe(server, fresh, 'rcon', rcon, rconStatusResult);
 }

src/lib/server/maintenance/recovery.ts

@@ -4,7 +4,8 @@ import { komodoConfigured } from '../config.js';
 import { nativeInstance } from '../backends/native.js';
 import { getServer, listServers } from '../fleet/servers.js';
 import { hasActiveProvisioning } from '../fleet/provisioning.js';
-import { inspectForMonitoring, rconForMonitoring, MonitoringChangedError } from '../fleet/health-observations.js';
+import { inspectForMonitoring, rconForMonitoring, monitoringStatus, monitoringIdentity, MonitoringChangedError } from '../fleet/health-observations.js';
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
 import { latestGameSubmission, telemetryEvidence, telemetryReport } from '../fleet/status-evidence.js';
 import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '../fleet/drift.js';
 import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
@@ -41,28 +42,34 @@ function transition(id: string, state: WatchdogState, summary: string) {
     event(id, summary);
 }
 function detect(server: string, kind: 'health' | 'drift', summary: string, details: WatchdogEvidence): WatchdogIncident {
+    const sampledAt = kind === 'health' ? details.observedAt ?? iso() : iso();
+    const sampledTime = Date.parse(sampledAt);
     const previous = active(server, kind);
     if (previous) {
         // Repeated UI scans must not manufacture a sustained failure.
-        const newSample = Date.now() - Date.parse(previous.last_seen_at) >= WATCHDOG_SCAN_MS;
+        const sampleGap = sampledTime - Date.parse(previous.last_seen_at);
         const actionable = (details as WatchdogEvidence & { actionable?: boolean }).actionable;
-        const priorActionable = (JSON.parse(previous.details) as { actionable?: boolean }).actionable;
+        const prior = JSON.parse(previous.details) as WatchdogEvidence & { actionable?: boolean };
+        const newSample = sampleGap > 0
+            && Date.now() - Date.parse(prior.countedAt ?? previous.last_seen_at) >= WATCHDOG_SCAN_MS;
+        const priorActionable = prior.actionable;
         // A successful routine/manual probe between watchdog samples breaks
         // the failure streak, even if our next probe happens to fail again.
         const after = latestGameSubmission(server);
         const successful = [getServer(server)?.last_seen_at, telemetryReport(server)?.receivedAt]
             .some(at => at && Date.parse(at) > Date.parse(previous.last_seen_at)
-                && Date.parse(at) <= Date.now() && (!after || Date.parse(at) > Date.parse(after)));
-        const reset = kind === 'health' && (successful || (actionable === true && priorActionable !== true));
+                && Date.parse(at) <= sampledTime && (!after || Date.parse(at) > Date.parse(after)));
+        const reset = kind === 'health' && (successful || (actionable === true && priorActionable !== true)
+            || prior.identity !== details.identity || sampleGap > WATCHDOG_SCAN_MS + ONLINE_MAX_MS);
         db().prepare('UPDATE watchdog_incidents SET details=?, last_seen_at=?, failures=?, first_seen_at=? WHERE id=?')
-            .run(JSON.stringify(details), newSample || reset ? iso() : previous.last_seen_at,
-                reset ? 1 : previous.failures + (newSample ? 1 : 0), reset ? iso() : previous.first_seen_at, previous.id);
+            .run(JSON.stringify({ ...details, countedAt: newSample || reset ? iso() : prior.countedAt ?? previous.last_seen_at }), newSample || reset ? sampledAt : previous.last_seen_at,
+                reset ? 1 : previous.failures + (newSample ? 1 : 0), reset ? sampledAt : previous.first_seen_at, previous.id);
         if (previous.state === 'open') transition(previous.id, 'open', summary);
         return incident(previous.id)!;
     }
     const id = randomUUID();
     db().prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at)
-        VALUES (?,?,?,'open',?,?,?,?)`).run(id, server, kind, summary, JSON.stringify(details), iso(), iso());
+        VALUES (?,?,?,'open',?,?,?,?)`).run(id, server, kind, summary, JSON.stringify({ ...details, countedAt: iso() }), sampledAt, sampledAt);
     event(id, summary);
     return incident(id)!;
 }
@@ -222,23 +229,29 @@ async function reconcileActions(): Promise<void> {
     }
 }
 
-interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; players?: number; source?: 'rcon' | 'telemetry' }
+interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; observedAt: string; identity: string; players?: number; source?: 'rcon' | 'telemetry' }
 async function observe(server: string, operationId?: string, fresh = false): Promise<Observation | null> {
     const epoch = latestGameSubmission(server);
     const suppressed = () => (operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server))
         || protectedWork(server) || getServer(server)?.desired_state !== 'running' || epoch !== latestGameSubmission(server);
-    const inspected = await inspectForMonitoring(server, fresh || !!operationId).catch(error => {
+    // Routine watchdog passes only consume the status monitor. Manual checks and
+    // recovery verification still obtain fresh, independently verified evidence.
+    const snapshot = !fresh && !operationId ? monitoringStatus(server) : null;
+    if (!fresh && !operationId && !snapshot) return null;
+    const inspected = snapshot?.container ?? await inspectForMonitoring(server, true).catch(error => {
         if (error instanceof MonitoringChangedError) return null;
         throw error;
     });
     if (!inspected) return null;
     const health = inspected.result;
     if (suppressed()) return null;
+    const identity = snapshot?.identity ?? monitoringIdentity(server);
+    const containerEvidence = { identity, observedAt: inspected.observedAt };
     const state = health.state?.toLowerCase() ?? null;
-    if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
+    if (!health.ok) return { ...containerEvidence, healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
     const telemetry = telemetryEvidence(server);
-    if (telemetry) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
-    const probed = await rconForMonitoring(server, fresh || !!operationId).catch(error => {
+    if (telemetry) return { identity, observedAt: telemetry.receivedAt, healthy: true, actionable: false, reason: 'Container healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
+    const probed = snapshot ? snapshot.game : await rconForMonitoring(server, true).catch(error => {
         if (error instanceof MonitoringChangedError) return null;
         throw error;
     });
@@ -246,9 +259,10 @@ async function observe(server: string, operationId?: string, fresh = false): Pro
     const rcon = probed.result;
     if (suppressed()) return null;
     const latest = telemetryEvidence(server);
-    if (rcon.ok || latest) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state,
+    const gameEvidence = { identity: snapshot?.identity ?? monitoringIdentity(server), observedAt: latest?.receivedAt ?? probed.observedAt };
+    if (rcon.ok || latest) return { ...gameEvidence, healthy: true, actionable: false, reason: 'Container healthy with fresh game status', containerState: state,
         players: latest?.players ?? (rcon.ok ? rcon.status.players : 0), source: latest ? 'telemetry' : 'rcon' };
-    return { healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
+    return { ...gameEvidence, healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
         reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
 }
 async function scanServer(server: string, forceScan: boolean): Promise<void> {
@@ -258,13 +272,14 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
     const observation = await observe(server, undefined, forceScan);
     // Probes can overlap an API request; recheck ownership before queuing anything.
     if (!observation || scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
+    if (observation.identity !== monitoringIdentity(server)) return;
     if (!scanSuppressed(server, 'health') && observation.healthy) {
         const current = active(server, 'health');
         if (current) transition(current.id, 'resolved', 'Fresh game status and Komodo health verified');
     } else if (!scanSuppressed(server, 'health')) {
         const problem = detect(server, 'health', observation.reason, observation);
         const startup = ['starting','restarting'].includes(observation.containerState ?? '');
-        const oldEnough = Date.now() - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
+        const oldEnough = Date.parse(observation.observedAt) - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
         if (problem.state === 'open' && observation.actionable && oldEnough && problem.failures >= 3 && maintenanceAllowed('health')) {
             const result = db().transaction(() => {
                 const next = requestWatchdogLifecycle(server, 'restart', problem.id);

src/lib/server/pipeline/worker.ts

@@ -6,7 +6,7 @@ import { emit } from './bus.js';
 import type { ServerStatusEntry } from './bus.js';
 import type { RconFailure } from '../integrations/rcon.js';
 import { getServer, listServers, setServerFields, type ServerRow } from '../fleet/servers.js';
-import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, MonitoringChangedError } from '../fleet/health-observations.js';
+import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, monitoringStatusWriter, MonitoringChangedError } from '../fleet/health-observations.js';
 import { restartSettling } from '../provisioning/index.js';
 import { serverObservation, observationReader } from '../fleet/status.js';
 import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
@@ -149,6 +149,9 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
     let updates: Partial<Record<keyof ServerRow, string | number | null>> = {};
     const flush = () => { setServerFields(srv.name, updates); updates = {}; };
     if (!isProvisioning(srv.name)) {
+        const publishObservation = monitoringStatusWriter(srv.name);
+        let gameObservation: Awaited<ReturnType<typeof rconForMonitoring>> | undefined;
+        const settling = restartSettling(srv.name);
         const epoch = latestGameSubmission(srv.name);
         const unchanged = () => {
             const current = getServer(srv.name);
@@ -157,7 +160,7 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
                 && current.native_instance_id === srv.native_instance_id && current.instance_generation === srv.instance_generation;
         };
         const inspectionStartedAt = new Date().toISOString();
-        const inspected = restartSettling(srv.name)
+        const inspected = settling
             ? { result: { ok: false as const, state: 'restarting', reason: 'Restart requested' }, observedAt: inspectionStartedAt }
             : await inspectForMonitoring(srv.name).catch(err => err instanceof MonitoringChangedError ? null
                 : { result: { ok: false as const, state: 'unknown', reason: (err as Error).message }, observedAt: inspectionStartedAt });
@@ -178,10 +181,11 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
                 flush();
                 const probeStartedAt = new Date().toISOString();
                 const probe = await (probeStatus ? probeStatus() : rconForMonitoring(srv.name)).catch(err => err instanceof MonitoringChangedError ? null
-                    : { result: { ok: false as const, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt });
+                    : { result: { ok: false as const, endpoint: null, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt });
                 if (!probe) return null;
                 const result = probe.result;
                 if (unchanged() && (getServer(srv.name)?.last_checked_at ?? '') <= probe.observedAt) {
+                    gameObservation = probe;
                     const now = probe.observedAt;
                     updates.last_checked_at = now;
                     updates.last_reply_snippet = result.rawReply ?? null;
@@ -196,6 +200,7 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
                     } else updates.last_error = encodeError(result.reason, result.detail);
                 }
             }
+            if (!settling && unchanged()) publishObservation(inspected, gameObservation);
         }
     } else {
         updates.last_error = null;

src/lib/watchdog.ts

@@ -17,6 +17,9 @@ export interface WatchdogIncident {
     operation_stage?: string | null;
 }
 export interface WatchdogEvidence {
+    observedAt?: string;
+    countedAt?: string;
+    identity?: string;
     reason?: string;
     containerState?: string | null;
     files?: DriftPlanFile[];