Preserve server status when batch probes miss the deadline

src/lib/server/fleet/health-observations.ts

@@ -23,6 +23,7 @@ interface Entry<T> {
 }
 
 export class MonitoringChangedError extends Error {}
+export class MonitoringSkippedError extends Error {}
 
 function identity(server: string, probe: Probe): Identity {
     const row = getServer(server);
@@ -199,6 +200,7 @@ export function prepareMonitoringRcon(servers: string[]): Map<string, () => Prom
                     const operation = withOperationSource('Status monitor', () => queueHostOperation(instance.host_id, 'rcon.status-batch', {
                         instances: instances.map(value => ({ instanceId: value.id, generation: value.generation }))
                     }));
+                    // Allow delivery and receipt overhead beyond the agent's 20-second budget.
                     const receipt = await waitHostOperationStatus(operation.id, 25_000);
                     if (receipt.state !== 'completed') throw new Error(receipt.error ?? 'Host status batch did not complete.');
                     return receipt.result as { results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> };
@@ -211,8 +213,12 @@ export function prepareMonitoringRcon(servers: string[]): Map<string, () => Prom
                 const receipt = await batch.result;
                 const replies = receipt.results.filter(value => value.instanceId === instance.id && value.generation === instance.generation);
                 if (replies.length !== 1 || typeof replies[0].reply?.text !== 'string') throw new Error('Host status batch omitted a valid instance result.');
+                if (replies[0].reply.reason === 'deadline') throw new MonitoringSkippedError('Host status batch budget expired before this server was checked.');
                 return agentRconStatusResult(replies[0].reply, endpoint);
-            } catch (error) { return { ok: false as const, endpoint, reason: 'timeout' as const, detail: (error as Error).message }; }
+            } catch (error) {
+                if (error instanceof MonitoringSkippedError) throw error;
+                return { ok: false as const, endpoint, reason: 'timeout' as const, detail: (error as Error).message };
+            }
         };
         // Observe before dispatch, including servers still waiting for their
         // queue slot. Consume failures too, so a failed batch is never retried

src/lib/server/pipeline/worker.ts

@@ -6,7 +6,7 @@ import { emit } from './bus.js';
 import type { ServerStatusEntry } from './bus.js';
 import type { RconFailure } from '../integrations/rcon.js';
 import { getServer, listServers, setServerFields, type ServerRow } from '../fleet/servers.js';
-import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, monitoringStatusWriter, MonitoringChangedError } from '../fleet/health-observations.js';
+import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, monitoringStatus, monitoringStatusWriter, MonitoringChangedError, MonitoringSkippedError } from '../fleet/health-observations.js';
 import { restartSettling } from '../provisioning/index.js';
 import { serverObservation, observationReader } from '../fleet/status.js';
 import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
@@ -149,6 +149,7 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
     let updates: Partial<Record<keyof ServerRow, string | number | null>> = {};
     const flush = () => { setServerFields(srv.name, updates); updates = {}; };
     if (!isProvisioning(srv.name)) {
+        const previousGame = monitoringStatus(srv.name)?.game;
         const publishObservation = monitoringStatusWriter(srv.name);
         let gameObservation: Awaited<ReturnType<typeof rconForMonitoring>> | undefined;
         const settling = restartSettling(srv.name);
@@ -180,8 +181,15 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
             } else if (unavailable || !telemetryEvidence(srv.name)) {
                 flush();
                 const probeStartedAt = new Date().toISOString();
-                const probe = await (probeStatus ? probeStatus() : rconForMonitoring(srv.name)).catch(err => err instanceof MonitoringChangedError ? null
-                    : { result: { ok: false as const, endpoint: null, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt });
+                const probe = await (probeStatus ? probeStatus() : rconForMonitoring(srv.name)).catch(err => {
+                    if (err instanceof MonitoringSkippedError) {
+                        // The container was checked, but game evidence retains its original age.
+                        if (unchanged()) publishObservation(inspected, previousGame);
+                        return null;
+                    }
+                    if (err instanceof MonitoringChangedError) return null;
+                    return { result: { ok: false as const, endpoint: null, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt };
+                });
                 if (!probe) return null;
                 const result = probe.result;
                 if (unchanged() && (getServer(srv.name)?.last_checked_at ?? '') <= probe.observedAt) {