Reuse heartbeat container state and batch host RCON status

src/lib/server/fleet/health-observations.ts

@@ -1,9 +1,10 @@
 import { config } from '../config.js';
 import { db } from '../db/index.js';
-import { getHost } from '../hosts/index.js';
-import { nativeInstance } from '../backends/native.js';
+import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js';
+import { nativeInstance, type NativeInstance } from '../backends/native.js';
 import { komodoInspectHealthy } from '../integrations/komodo.js';
-import { rconStatusResult } from '../integrations/rcon.js';
+import { rconStatusResult, agentRconStatusResult, type AgentRconReply } from '../integrations/rcon.js';
+import { withOperationSource } from '../hosts/operation-context.js';
 import { getServer } from './servers.js';
 import { latestGameSubmission } from './lifecycle-evidence.js';
 
@@ -99,8 +100,83 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
 const inspections = new Map<string, Entry<Awaited<ReturnType<typeof komodoInspectHealthy>>>>();
 const rcon = new Map<string, Entry<Awaited<ReturnType<typeof rconStatusResult>>>>();
 
+/** Keep the actual sample time: a later heartbeat must not renew old evidence. */
+export function heartbeatInspection(server: string): Observation<Awaited<ReturnType<typeof komodoInspectHealthy>>> | null {
+    const row = getServer(server);
+    if (!row?.native_instance_id || !row.host_id) return null;
+    const host = getHost(row.host_id);
+    if (!host?.connected || !host.inventory.capabilities?.includes('instance.heartbeat-state')) return null;
+    const state = host.inventory.instanceStates?.find(state => state.instanceId === row.native_instance_id && state.generation === row.instance_generation);
+    if (!state) return null;
+    const at = Date.parse(state.observedAt);
+    const lifecycle = db().prepare(`SELECT MAX(MAX(submitted_at,COALESCE(finished_at,submitted_at))) AS at
+        FROM server_deployments WHERE server=? AND server_deleted_at IS NULL`).get(server) as { at: string | null };
+    if (!reusable(at) || at < Date.parse(row.created_at) || at < Date.parse(lifecycle.at ?? '')
+        || at < Date.parse(latestGameSubmission(server) ?? '') || at < Date.parse(row.container_checked_at ?? '')) return null;
+    return { observedAt: new Date(at).toISOString(), result: state.healthy
+        ? { ok: true, state: state.state } : { ok: false, state: state.state, reason: `Container ${state.state}` } };
+}
+
+type RconObservation = Observation<Awaited<ReturnType<typeof rconStatusResult>>>;
+
+/** Prime a whole host before the per-host queue admits just its first two servers. */
+export function prepareMonitoringRcon(servers: string[]): Map<string, () => Promise<RconObservation>> {
+    const probes = new Map<string, () => Promise<RconObservation>>();
+    const batches = new Map<string, { instances: NativeInstance[]; result: Promise<{ results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> }> }>();
+    for (const server of servers) {
+        const instance = nativeInstance(server);
+        if (!instance) continue;
+        const host = getHost(instance.host_id);
+        if (!host?.connected || !host.inventory.capabilities?.includes('rcon.status-batch')) continue;
+        const container = heartbeatInspection(server);
+        if (container && !container.result.ok && container.result.state !== 'unknown') continue;
+        const before = identity(server, 'rcon');
+        const request = async () => {
+            let batch = batches.get(instance.host_id);
+            if (!batch) {
+                const instances: NativeInstance[] = [];
+                // observe() invokes each request in the same microtask turn;
+                // enqueue afterward so cached or shared probes need no batch slot.
+                const result = Promise.resolve().then(async () => {
+                    const operation = withOperationSource('Status monitor', () => queueHostOperation(instance.host_id, 'rcon.status-batch', {
+                        instances: instances.map(value => ({ instanceId: value.id, generation: value.generation }))
+                    }));
+                    const receipt = await waitHostOperationStatus(operation.id, 25_000);
+                    if (receipt.state !== 'completed') throw new Error(receipt.error ?? 'Host status batch did not complete.');
+                    return receipt.result as { results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> };
+                });
+                batch = { instances, result }; batches.set(instance.host_id, batch);
+            }
+            batch.instances.push(instance);
+            const endpoint = { host: `agent:${instance.host_id}`, port: instance.port };
+            try {
+                const receipt = await batch.result;
+                const replies = receipt.results.filter(value => value.instanceId === instance.id && value.generation === instance.generation);
+                if (replies.length !== 1 || typeof replies[0].reply?.text !== 'string') throw new Error('Host status batch omitted a valid instance result.');
+                return agentRconStatusResult(replies[0].reply, endpoint);
+            } catch (error) { return { ok: false as const, endpoint, reason: 'timeout' as const, detail: (error as Error).message }; }
+        };
+        // Observe before dispatch, including servers still waiting for their
+        // queue slot. Consume failures too, so a failed batch is never retried
+        // once per server or left as an unhandled rejected promise.
+        const pending = observe(server, false, 'rcon', rcon, request).then(value => ({ value }), error => ({ error }));
+        probes.set(server, async () => {
+            const result = await pending;
+            if ('error' in result) throw result.error;
+            const current = identity(server, 'rcon');
+            if (!sameIdentity(before, current) || newerEvidence(current, Date.parse(result.value.observedAt))) {
+                throw new MonitoringChangedError('Server changed before consuming the host status batch.');
+            }
+            return result.value;
+        });
+    }
+    return probes;
+}
+
 /** Routine monitoring only. Manual diagnostics and disruption gates use fresh probes directly. */
 export function inspectForMonitoring(server: string, fresh = false) {
+    const heartbeat = !fresh && heartbeatInspection(server);
+    if (heartbeat) return Promise.resolve(heartbeat);
     return observe(server, fresh, 'inspect', inspections, komodoInspectHealthy);
 }
 export function rconForMonitoring(server: string, fresh = false) {