Reuse heartbeat container state and batch host RCON status

src/lib/hosts.ts

@@ -70,6 +70,10 @@ export interface HostInventory {
     agentUpdate?: HostAgentUpdate;
     instanceResources?: InstanceResources[];
     instanceResourcesError?: string;
+    instanceStates?: InstanceState[];
+}
+export interface InstanceState {
+    instanceId: string; generation: number; state: string; healthy: boolean; observedAt: string;
 }
 export interface InstanceResources {
     instanceId: string; generation: number; uptimeSeconds: number; cpuCores: number; memoryBytes: number;

src/lib/server/fleet/health-observations.ts

@@ -1,9 +1,10 @@
 import { config } from '../config.js';
 import { db } from '../db/index.js';
-import { getHost } from '../hosts/index.js';
-import { nativeInstance } from '../backends/native.js';
+import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js';
+import { nativeInstance, type NativeInstance } from '../backends/native.js';
 import { komodoInspectHealthy } from '../integrations/komodo.js';
-import { rconStatusResult } from '../integrations/rcon.js';
+import { rconStatusResult, agentRconStatusResult, type AgentRconReply } from '../integrations/rcon.js';
+import { withOperationSource } from '../hosts/operation-context.js';
 import { getServer } from './servers.js';
 import { latestGameSubmission } from './lifecycle-evidence.js';
 
@@ -99,8 +100,83 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
 const inspections = new Map<string, Entry<Awaited<ReturnType<typeof komodoInspectHealthy>>>>();
 const rcon = new Map<string, Entry<Awaited<ReturnType<typeof rconStatusResult>>>>();
 
+/** Keep the actual sample time: a later heartbeat must not renew old evidence. */
+export function heartbeatInspection(server: string): Observation<Awaited<ReturnType<typeof komodoInspectHealthy>>> | null {
+    const row = getServer(server);
+    if (!row?.native_instance_id || !row.host_id) return null;
+    const host = getHost(row.host_id);
+    if (!host?.connected || !host.inventory.capabilities?.includes('instance.heartbeat-state')) return null;
+    const state = host.inventory.instanceStates?.find(state => state.instanceId === row.native_instance_id && state.generation === row.instance_generation);
+    if (!state) return null;
+    const at = Date.parse(state.observedAt);
+    const lifecycle = db().prepare(`SELECT MAX(MAX(submitted_at,COALESCE(finished_at,submitted_at))) AS at
+        FROM server_deployments WHERE server=? AND server_deleted_at IS NULL`).get(server) as { at: string | null };
+    if (!reusable(at) || at < Date.parse(row.created_at) || at < Date.parse(lifecycle.at ?? '')
+        || at < Date.parse(latestGameSubmission(server) ?? '') || at < Date.parse(row.container_checked_at ?? '')) return null;
+    return { observedAt: new Date(at).toISOString(), result: state.healthy
+        ? { ok: true, state: state.state } : { ok: false, state: state.state, reason: `Container ${state.state}` } };
+}
+
+type RconObservation = Observation<Awaited<ReturnType<typeof rconStatusResult>>>;
+
+/** Prime a whole host before the per-host queue admits just its first two servers. */
+export function prepareMonitoringRcon(servers: string[]): Map<string, () => Promise<RconObservation>> {
+    const probes = new Map<string, () => Promise<RconObservation>>();
+    const batches = new Map<string, { instances: NativeInstance[]; result: Promise<{ results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> }> }>();
+    for (const server of servers) {
+        const instance = nativeInstance(server);
+        if (!instance) continue;
+        const host = getHost(instance.host_id);
+        if (!host?.connected || !host.inventory.capabilities?.includes('rcon.status-batch')) continue;
+        const container = heartbeatInspection(server);
+        if (container && !container.result.ok && container.result.state !== 'unknown') continue;
+        const before = identity(server, 'rcon');
+        const request = async () => {
+            let batch = batches.get(instance.host_id);
+            if (!batch) {
+                const instances: NativeInstance[] = [];
+                // observe() invokes each request in the same microtask turn;
+                // enqueue afterward so cached or shared probes need no batch slot.
+                const result = Promise.resolve().then(async () => {
+                    const operation = withOperationSource('Status monitor', () => queueHostOperation(instance.host_id, 'rcon.status-batch', {
+                        instances: instances.map(value => ({ instanceId: value.id, generation: value.generation }))
+                    }));
+                    const receipt = await waitHostOperationStatus(operation.id, 25_000);
+                    if (receipt.state !== 'completed') throw new Error(receipt.error ?? 'Host status batch did not complete.');
+                    return receipt.result as { results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> };
+                });
+                batch = { instances, result }; batches.set(instance.host_id, batch);
+            }
+            batch.instances.push(instance);
+            const endpoint = { host: `agent:${instance.host_id}`, port: instance.port };
+            try {
+                const receipt = await batch.result;
+                const replies = receipt.results.filter(value => value.instanceId === instance.id && value.generation === instance.generation);
+                if (replies.length !== 1 || typeof replies[0].reply?.text !== 'string') throw new Error('Host status batch omitted a valid instance result.');
+                return agentRconStatusResult(replies[0].reply, endpoint);
+            } catch (error) { return { ok: false as const, endpoint, reason: 'timeout' as const, detail: (error as Error).message }; }
+        };
+        // Observe before dispatch, including servers still waiting for their
+        // queue slot. Consume failures too, so a failed batch is never retried
+        // once per server or left as an unhandled rejected promise.
+        const pending = observe(server, false, 'rcon', rcon, request).then(value => ({ value }), error => ({ error }));
+        probes.set(server, async () => {
+            const result = await pending;
+            if ('error' in result) throw result.error;
+            const current = identity(server, 'rcon');
+            if (!sameIdentity(before, current) || newerEvidence(current, Date.parse(result.value.observedAt))) {
+                throw new MonitoringChangedError('Server changed before consuming the host status batch.');
+            }
+            return result.value;
+        });
+    }
+    return probes;
+}
+
 /** Routine monitoring only. Manual diagnostics and disruption gates use fresh probes directly. */
 export function inspectForMonitoring(server: string, fresh = false) {
+    const heartbeat = !fresh && heartbeatInspection(server);
+    if (heartbeat) return Promise.resolve(heartbeat);
     return observe(server, fresh, 'inspect', inspections, komodoInspectHealthy);
 }
 export function rconForMonitoring(server: string, fresh = false) {

src/lib/server/hosts/index.ts

@@ -254,6 +254,15 @@ export function validateInventory(value: unknown): HostInventory {
     }
     if (inventory.cpuUsagePercent !== undefined && (typeof inventory.cpuUsagePercent !== 'number' || !Number.isFinite(inventory.cpuUsagePercent) || inventory.cpuUsagePercent < 0 || inventory.cpuUsagePercent > 100)) throw new Error('Invalid inventory cpuUsagePercent.');
     validateInstanceResources(inventory);
+    if (inventory.instanceStates !== undefined && (!Array.isArray(inventory.instanceStates) || inventory.instanceStates.length > 1000
+        || inventory.instanceStates.some(state => !state || typeof state !== 'object'
+            || typeof state.instanceId !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(state.instanceId)
+            || !Number.isSafeInteger(state.generation) || state.generation < 1
+            || typeof state.state !== 'string' || !/^[a-z-]{1,40}$/.test(state.state)
+            || typeof state.healthy !== 'boolean' || state.healthy && state.state !== 'running'
+            || typeof state.observedAt !== 'string' || state.observedAt.length > 40 || !Number.isFinite(Date.parse(state.observedAt))))) {
+        throw new Error('Invalid instance state inventory.');
+    }
     for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
     if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
     if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
@@ -486,9 +495,9 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
         WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId)) return [];
     // Accepted/running/unknown commands keep the same immutable ID. The agent
     // consults its durable journal before acknowledging or executing a replay.
-    const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','files.manifest','logs.read')
+    const readOnly = `(kind IN ('host.inspect','instance.inspect','rcon.status-batch','files.read','files.list','files.stat','files.manifest','logs.read')
         OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
-    const healthOnly = `(kind='instance.inspect' OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
+    const healthOnly = `(kind IN ('instance.inspect','rcon.status-batch') OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
     const healthLane = host.inventory.capabilities?.includes('operations.health-lane');
     const laneFilter = lane === 'all' ? '' : lane === 'health' ? ` AND ${healthOnly}`
         : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`;

src/lib/server/hosts/observations.ts

@@ -1,4 +1,4 @@
-export const observationKinds = ['host.inspect', 'instance.inspect', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read'];
+export const observationKinds = ['host.inspect', 'instance.inspect', 'rcon.status-batch', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read'];
 const commands = ['status', 'getstatus', 'version'];
 
 export function safeOperationTarget(kind: string, value: string | null): string | null {

src/lib/server/integrations/rcon.ts

@@ -367,6 +367,30 @@ export function formatRawReply(buf: Buffer): string {
     return lines.join('\n');
 }
 
+export interface AgentRconReply { ok?: boolean; reason?: string; text: string }
+
+export function agentRconStatusResult(reply: AgentRconReply, endpoint: { host: string; port: number }): RconStatusResult {
+    if (reply.ok === false) {
+        const failures: Record<string, { reason: RconFailure; detail: string }> = {
+            unconfigured: { reason: 'no-config', detail: 'Agent RCON password is not configured.' },
+            unreachable: { reason: 'send-error', detail: 'Agent could not reach the local RCON endpoint.' },
+            timeout: { reason: 'timeout', detail: 'Agent received no reply from the local RCON endpoint.' },
+            authentication: { reason: 'bad-reply', detail: 'wrong RCON password' },
+            'invalid-reply': { reason: 'bad-reply', detail: 'Agent RCON reply is missing the 0xff prefix.' }
+        };
+        return { ok: false, endpoint, ...(failures[reply.reason ?? ''] ?? {
+            reason: 'bad-reply' as const, detail: 'Agent RCON query failed.'
+        }) };
+    }
+    const players = parsePlayerCount(reply.text);
+    if (players === null) return { ok: false, endpoint, reason: 'bad-reply', detail: 'Agent returned an unrecognized RCON status.', rawReply: reply.text.slice(0, 4000) };
+    const info = parseInfoLine(reply.text); const header = parseStatusHeader(reply.text);
+    return { ok: true, endpoint, status: { players,
+        capacity: positiveCapacity(info.sv_maxclients ?? header.sv_maxclients),
+        map: info.mapname ?? info.sv_mapname ?? header.map ?? header.mapname ?? header.sv_mapname ?? null,
+        gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null, roster: parsePlayerRoster(reply.text) }, rawReply: reply.text.slice(0, 4000) };
+}
+
 /**
  * Full RCON status probe with structured diagnostics. Callers get either
  * (ok, endpoint, status) or (ok:false, endpoint?, reason, detail). This is
@@ -383,25 +407,7 @@ export async function rconStatusResult(server: string): Promise<RconStatusResult
         const request = (async (): Promise<RconStatusResult> => {
             try {
                 const reply = await nativeRequest<{ ok?: boolean; reason?: string; text: string }>(native, 'rcon.execute', { command: 'status' });
-                if (reply.ok === false) {
-                    const failures: Record<string, { reason: RconFailure; detail: string }> = {
-                        unconfigured: { reason: 'no-config', detail: 'Agent RCON password is not configured.' },
-                        unreachable: { reason: 'send-error', detail: 'Agent could not reach the local RCON endpoint.' },
-                        timeout: { reason: 'timeout', detail: 'Agent received no reply from the local RCON endpoint.' },
-                        authentication: { reason: 'bad-reply', detail: 'wrong RCON password' },
-                        'invalid-reply': { reason: 'bad-reply', detail: 'Agent RCON reply is missing the 0xff prefix.' }
-                    };
-                    return { ok: false, endpoint, ...(failures[reply.reason ?? ''] ?? {
-                        reason: 'bad-reply' as const, detail: 'Agent RCON query failed.'
-                    }) };
-                }
-                const players = parsePlayerCount(reply.text);
-                if (players === null) return { ok: false, endpoint, reason: 'bad-reply', detail: 'Agent returned an unrecognized RCON status.', rawReply: reply.text.slice(0, 4000) };
-                const info = parseInfoLine(reply.text); const header = parseStatusHeader(reply.text);
-                return { ok: true, endpoint, status: { players,
-                    capacity: positiveCapacity(info.sv_maxclients ?? header.sv_maxclients),
-                    map: info.mapname ?? info.sv_mapname ?? header.map ?? header.mapname ?? header.sv_mapname ?? null,
-                    gametype: info.g_gametype ?? header.g_gametype ?? header.gametype ?? null, roster: parsePlayerRoster(reply.text) }, rawReply: reply.text.slice(0, 4000) };
+                return agentRconStatusResult(reply, endpoint);
             } catch (error) { return { ok: false, endpoint, reason: 'timeout', detail: (error as Error).message }; }
         })().finally(() => statusInFlight.delete(key));
         statusInFlight.set(key, request); return request;

src/lib/server/pipeline/worker.ts

@@ -6,7 +6,7 @@ import { emit } from './bus.js';
 import type { ServerStatusEntry } from './bus.js';
 import type { RconFailure } from '../integrations/rcon.js';
 import { getServer, listServers, setServerFields, type ServerRow } from '../fleet/servers.js';
-import { inspectForMonitoring, rconForMonitoring, MonitoringChangedError } from '../fleet/health-observations.js';
+import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, MonitoringChangedError } from '../fleet/health-observations.js';
 import { restartSettling } from '../provisioning/index.js';
 import { serverObservation, observationReader } from '../fleet/status.js';
 import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
@@ -96,7 +96,7 @@ function drainStatusQueue(host: string, queue: StatusQueue): void {
     if (!queue.active && !queue.queued.length) statusQueues.delete(host);
 }
 
-function queueStatus(srv: ServerRow): Promise<number | null> {
+function queueStatus(srv: ServerRow, probe?: () => ReturnType<typeof rconForMonitoring>): Promise<number | null> {
     const server = srv.name;
     const pending = pendingStatus.get(server);
     if (pending) return pending;
@@ -110,7 +110,7 @@ function queueStatus(srv: ServerRow): Promise<number | null> {
     queue.queued.push(async () => {
         let capacity: number | null = null;
         try {
-            capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server));
+            capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server, probe));
             // Publish this server's latest evidence without waiting for the fleet.
             const current = getServer(server);
             if (current) publishStatus(statusEntry(current));
@@ -124,7 +124,9 @@ function queueStatus(srv: ServerRow): Promise<number | null> {
 export async function fanoutStatus(): Promise<void> {
     const servers = listServers();
     if (!servers.length) return;
-    const results = await Promise.all(servers.map(queueStatus));
+    const probes = prepareMonitoringRcon(servers.filter(server => !pendingStatus.has(server.name)
+        && !isProvisioning(server.name) && !restartSettling(server.name) && !telemetryEvidence(server.name)).map(server => server.name));
+    const results = await Promise.all(servers.map(server => queueStatus(server, probes.get(server.name))));
     const capacities = new Map(servers.map((server, i) => [server.name, results[i]]));
     try {
         // Metrics remain complete fleet collections, using freshness at capture
@@ -140,7 +142,7 @@ export async function fanoutStatus(): Promise<void> {
     } catch (err) { console.error('[worker] metrics error:', err); }
 }
 
-async function refreshServerStatus(server: string): Promise<number | null> {
+async function refreshServerStatus(server: string, probeStatus?: () => ReturnType<typeof rconForMonitoring>): Promise<number | null> {
     const srv = getServer(server);
     if (!srv) return null;
     let capacity: number | null = null;
@@ -175,7 +177,7 @@ async function refreshServerStatus(server: string): Promise<number | null> {
             } else if (unavailable || !telemetryEvidence(srv.name)) {
                 flush();
                 const probeStartedAt = new Date().toISOString();
-                const probe = await rconForMonitoring(srv.name).catch(err => err instanceof MonitoringChangedError ? null
+                const probe = await (probeStatus ? probeStatus() : rconForMonitoring(srv.name)).catch(err => err instanceof MonitoringChangedError ? null
                     : { result: { ok: false as const, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt });
                 if (!probe) return null;
                 const result = probe.result;