Share recent health observations between background monitors

src/lib/server/fleet/health-observations.ts

@@ -0,0 +1,108 @@
+import { config } from '../config.js';
+import { db } from '../db/index.js';
+import { getHost } from '../hosts/index.js';
+import { nativeInstance } from '../backends/native.js';
+import { komodoInspectHealthy } from '../integrations/komodo.js';
+import { rconStatusResult } from '../integrations/rcon.js';
+import { getServer } from './servers.js';
+import { latestGameSubmission } from './lifecycle-evidence.js';
+
+const MAX_AGE_MS = 30_000;
+const MAX_SERVERS = 1024;
+type Probe = 'inspect' | 'rcon';
+interface Observation<T> { result: T; observedAt: string }
+interface Identity { key: string; port: number | null; checkedAt: string | null }
+interface Entry<T> {
+    identity: Identity;
+    startedAt: number;
+    pending?: Promise<Observation<T>>;
+    observation?: Observation<T>;
+}
+
+export class MonitoringChangedError extends Error {}
+
+function identity(server: string, probe: Probe): Identity {
+    const row = getServer(server);
+    if (!row || row.destroyed_at) throw new MonitoringChangedError('Server is no longer registered; monitoring evidence is unavailable.');
+    const native = row.native_instance_id ? nativeInstance(server) : null;
+    if (row.native_instance_id && (!native || getHost(native.host_id)?.connectionState !== 'online')) {
+        throw new Error('Host agent is offline; monitoring evidence is unavailable.');
+    }
+    // Stop requests also invalidate observations, even though they do not
+    // establish a new game-telemetry epoch in latestGameSubmission().
+    const lifecycle = db().prepare(`SELECT id, action, submitted_at FROM server_deployments
+        WHERE server=? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) ?? null;
+    return {
+        key: JSON.stringify([row.created_at, row.desired_state, latestGameSubmission(server), lifecycle,
+            native ? [native.host_id, native.id, native.generation] : null,
+            native ? null : [row.komodo_type, row.komodo_name, row.komodo_server, config.komodo, config.httpTimeoutMs],
+            probe === 'rcon' ? [row.rcon_host, config.rcon] : null]),
+        port: probe === 'rcon' ? native?.port ?? row.game_port : null,
+        checkedAt: probe === 'rcon' ? row.last_checked_at : row.container_checked_at
+    };
+}
+
+function sameIdentity(left: Identity, right: Identity): boolean {
+    return left.key === right.key && left.port === right.port;
+}
+function newerEvidence(current: Identity, startedAt: number): boolean {
+    return Date.parse(current.checkedAt ?? '') > startedAt;
+}
+function reusable(startedAt: number): boolean {
+    const age = Date.now() - startedAt;
+    return age >= 0 && age < MAX_AGE_MS;
+}
+
+async function observe<T extends { ok: boolean }>(server: string, fresh: boolean, probe: Probe,
+    entries: Map<string, Entry<T>>, request: (server: string) => Promise<T>): Promise<Observation<T>> {
+    let current: Identity;
+    try { current = identity(server, probe); }
+    catch (error) { entries.delete(server); throw error; }
+    const saved = entries.get(server);
+    const savedAt = saved?.observation ? Date.parse(saved.observation.observedAt) : saved?.startedAt ?? 0;
+    if (!fresh && saved && sameIdentity(saved.identity, current) && !newerEvidence(current, savedAt)) {
+        if (saved.pending) return saved.pending;
+        if (saved.observation && reusable(saved.startedAt)) return saved.observation;
+    }
+
+    // Replace the previous slot before dispatch, including for forced probes.
+    // A forced failure must not leave an older successful result reusable.
+    const entry: Entry<T> = { identity: current, startedAt: Date.now() };
+    entries.delete(server);
+    entries.set(server, entry);
+    while (entries.size > MAX_SERVERS) entries.delete(entries.keys().next().value!);
+    const pending = Promise.resolve().then(() => request(server)).then(result => {
+        const latest = identity(server, probe);
+        // Legacy RCON may discover and save its first port during this request.
+        // Accept that enrichment only when all other identity fields agree.
+        const endpoint = 'endpoint' in result ? result.endpoint : null;
+        const learnedPort = probe === 'rcon' && !current.port && result.ok
+            && endpoint !== null && typeof endpoint === 'object' && 'port' in endpoint && endpoint.port === latest.port;
+        if (current.key !== latest.key || (current.port !== latest.port && !learnedPort)
+            || newerEvidence(latest, entry.startedAt)) {
+            throw new MonitoringChangedError('Server changed during monitoring; the previous observation was discarded.');
+        }
+        const observation = { result, observedAt: new Date().toISOString() };
+        if (entries.get(server) === entry && result.ok && reusable(entry.startedAt)) {
+            entry.identity = latest;
+            entry.observation = observation;
+        }
+        return observation;
+    }).finally(() => {
+        entry.pending = undefined;
+        if (entries.get(server) === entry && !entry.observation) entries.delete(server);
+    });
+    entry.pending = pending;
+    return pending;
+}
+
+const inspections = new Map<string, Entry<Awaited<ReturnType<typeof komodoInspectHealthy>>>>();
+const rcon = new Map<string, Entry<Awaited<ReturnType<typeof rconStatusResult>>>>();
+
+/** Routine monitoring only. Manual diagnostics and disruption gates use fresh probes directly. */
+export function inspectForMonitoring(server: string, fresh = false) {
+    return observe(server, fresh, 'inspect', inspections, komodoInspectHealthy);
+}
+export function rconForMonitoring(server: string, fresh = false) {
+    return observe(server, fresh, 'rcon', rcon, rconStatusResult);
+}