import { config } from '../config.js'; import { db } from '../db/index.js'; import { getHost } from '../hosts/index.js'; import { nativeInstance } from '../backends/native.js'; import { komodoInspectHealthy } from '../integrations/komodo.js'; import { rconStatusResult } from '../integrations/rcon.js'; import { getServer } from './servers.js'; import { latestGameSubmission } from './lifecycle-evidence.js'; const MAX_AGE_MS = 30_000; const MAX_SERVERS = 1024; type Probe = 'inspect' | 'rcon'; interface Observation { result: T; observedAt: string } interface Identity { key: string; port: number | null; checkedAt: string | null } interface Entry { identity: Identity; startedAt: number; pending?: Promise>; observation?: Observation; } export class MonitoringChangedError extends Error {} function identity(server: string, probe: Probe): Identity { const row = getServer(server); if (!row || row.destroyed_at) throw new MonitoringChangedError('Server is no longer registered; monitoring evidence is unavailable.'); const native = row.native_instance_id ? nativeInstance(server) : null; if (row.native_instance_id && (!native || getHost(native.host_id)?.connectionState !== 'online')) { throw new Error('Host agent is offline; monitoring evidence is unavailable.'); } // Stop requests also invalidate observations, even though they do not // establish a new game-telemetry epoch in latestGameSubmission(). const lifecycle = db().prepare(`SELECT id, action, submitted_at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) ?? null; return { key: JSON.stringify([row.created_at, row.desired_state, latestGameSubmission(server), lifecycle, native ? [native.host_id, native.id, native.generation] : null, native ? null : [row.komodo_type, row.komodo_name, row.komodo_server, config.komodo, config.httpTimeoutMs], probe === 'rcon' ? [row.rcon_host, config.rcon] : null]), port: probe === 'rcon' ? native?.port ?? row.game_port : null, checkedAt: probe === 'rcon' ? row.last_checked_at : row.container_checked_at }; } function sameIdentity(left: Identity, right: Identity): boolean { return left.key === right.key && left.port === right.port; } function newerEvidence(current: Identity, startedAt: number): boolean { return Date.parse(current.checkedAt ?? '') > startedAt; } function reusable(startedAt: number): boolean { const age = Date.now() - startedAt; return age >= 0 && age < MAX_AGE_MS; } async function observe(server: string, fresh: boolean, probe: Probe, entries: Map>, request: (server: string) => Promise): Promise> { let current: Identity; try { current = identity(server, probe); } catch (error) { entries.delete(server); throw error; } const saved = entries.get(server); const savedAt = saved?.observation ? Date.parse(saved.observation.observedAt) : saved?.startedAt ?? 0; if (!fresh && saved && sameIdentity(saved.identity, current) && !newerEvidence(current, savedAt)) { if (saved.pending) return saved.pending; if (saved.observation && reusable(saved.startedAt)) return saved.observation; } // Replace the previous slot before dispatch, including for forced probes. // A forced failure must not leave an older successful result reusable. const entry: Entry = { identity: current, startedAt: Date.now() }; entries.delete(server); entries.set(server, entry); while (entries.size > MAX_SERVERS) entries.delete(entries.keys().next().value!); const pending = Promise.resolve().then(() => request(server)).then(result => { const latest = identity(server, probe); // Legacy RCON may discover and save its first port during this request. // Accept that enrichment only when all other identity fields agree. const endpoint = 'endpoint' in result ? result.endpoint : null; const learnedPort = probe === 'rcon' && !current.port && result.ok && endpoint !== null && typeof endpoint === 'object' && 'port' in endpoint && endpoint.port === latest.port; if (current.key !== latest.key || (current.port !== latest.port && !learnedPort) || newerEvidence(latest, entry.startedAt)) { throw new MonitoringChangedError('Server changed during monitoring; the previous observation was discarded.'); } const observation = { result, observedAt: new Date().toISOString() }; if (entries.get(server) === entry && result.ok && reusable(entry.startedAt)) { entry.identity = latest; entry.observation = observation; } return observation; }).finally(() => { entry.pending = undefined; if (entries.get(server) === entry && !entry.observation) entries.delete(server); }); entry.pending = pending; return pending; } const inspections = new Map>>>(); const rcon = new Map>>>(); /** Routine monitoring only. Manual diagnostics and disruption gates use fresh probes directly. */ export function inspectForMonitoring(server: string, fresh = false) { return observe(server, fresh, 'inspect', inspections, komodoInspectHealthy); } export function rconForMonitoring(server: string, fresh = false) { return observe(server, fresh, 'rcon', rcon, rconStatusResult); }