Use status monitor evidence for routine watchdog health checks

src/lib/server/fleet/health-observations.ts

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto';
 import { config } from '../config.js';
 import { db } from '../db/index.js';
 import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js';
@@ -7,6 +8,7 @@ import { rconStatusResult, agentRconStatusResult, type AgentRconReply } from '..
 import { withOperationSource } from '../hosts/operation-context.js';
 import { getServer } from './servers.js';
 import { latestGameSubmission } from './lifecycle-evidence.js';
+import { ONLINE_MAX_MS } from '../../server-freshness.js';
 
 const MAX_AGE_MS = 30_000;
 const MAX_SERVERS = 1024;
@@ -100,6 +102,62 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
 const inspections = new Map<string, Entry<Awaited<ReturnType<typeof komodoInspectHealthy>>>>();
 const rcon = new Map<string, Entry<Awaited<ReturnType<typeof rconStatusResult>>>>();
 
+interface MonitoringStatus {
+    identity: string;
+    container: Observation<Awaited<ReturnType<typeof komodoInspectHealthy>>>;
+    game?: Observation<Awaited<ReturnType<typeof rconStatusResult>>>;
+}
+// Keep failed samples too, independently of the successful-probe cache. Reading
+// this snapshot must never dispatch a request or renew an observation's age.
+const statuses = new Map<string, { snapshot?: MonitoringStatus }>();
+
+export function monitoringIdentity(server: string): string {
+    const container = identity(server, 'inspect');
+    const game = identity(server, 'rcon');
+    // Identity includes connection configuration; persist only its fingerprint.
+    return createHash('sha256').update(JSON.stringify([container.key, game.key, game.port])).digest('hex');
+}
+
+/** Capture ownership before a status pass; publish only evidence for that instance. */
+export function monitoringStatusWriter(server: string) {
+    let before: string;
+    try { before = identity(server, 'inspect').key; }
+    catch { return () => {}; }
+    const slot = { snapshot: statuses.get(server)?.snapshot };
+    statuses.delete(server);
+    statuses.set(server, slot);
+    while (statuses.size > MAX_SERVERS) statuses.delete(statuses.keys().next().value!);
+    return (container: MonitoringStatus['container'], game?: MonitoringStatus['game']) => {
+        // A forced check or newer pass supersedes this writer, even while its
+        // network requests are still in flight.
+        if (statuses.get(server) !== slot) return;
+        try {
+            if (identity(server, 'inspect').key !== before) return;
+            const current = slot.snapshot;
+            if (current && current.container.observedAt > container.observedAt) return;
+            slot.snapshot = { identity: monitoringIdentity(server), container, game };
+        } catch { statuses.delete(server); }
+    };
+}
+
+/** Latest completed status-monitor pass, with stale or superseded evidence excluded. */
+export function monitoringStatus(server: string): MonitoringStatus | null {
+    const saved = statuses.get(server)?.snapshot;
+    if (!saved) return null;
+    const fresh = (at: string) => {
+        const age = Date.now() - Date.parse(at);
+        return age >= 0 && age <= ONLINE_MAX_MS;
+    };
+    try {
+        if (saved.identity !== monitoringIdentity(server)
+            || !fresh(saved.container.observedAt)
+            || newerEvidence(identity(server, 'inspect'), Date.parse(saved.container.observedAt))) return null;
+        const game = saved.game && fresh(saved.game.observedAt)
+            && !newerEvidence(identity(server, 'rcon'), Date.parse(saved.game.observedAt)) ? saved.game : undefined;
+        return { ...saved, game };
+    } catch { return null; }
+}
+
 /** Keep the actual sample time: a later heartbeat must not renew old evidence. */
 export function heartbeatInspection(server: string): Observation<Awaited<ReturnType<typeof komodoInspectHealthy>>> | null {
     const row = getServer(server);
@@ -175,10 +233,12 @@ export function prepareMonitoringRcon(servers: string[]): Map<string, () => Prom
 
 /** Routine monitoring only. Manual diagnostics and disruption gates use fresh probes directly. */
 export function inspectForMonitoring(server: string, fresh = false) {
+    if (fresh) statuses.delete(server);
     const heartbeat = !fresh && heartbeatInspection(server);
     if (heartbeat) return Promise.resolve(heartbeat);
     return observe(server, fresh, 'inspect', inspections, komodoInspectHealthy);
 }
 export function rconForMonitoring(server: string, fresh = false) {
+    if (fresh) statuses.delete(server);
     return observe(server, fresh, 'rcon', rcon, rconStatusResult);
 }