Preserve server status when batch probes miss the deadline

src/lib/server/fleet/health-observations.ts

@@ -23,6 +23,7 @@ interface Entry<T> {
 }
 
 export class MonitoringChangedError extends Error {}
+export class MonitoringSkippedError extends Error {}
 
 function identity(server: string, probe: Probe): Identity {
     const row = getServer(server);
@@ -199,6 +200,7 @@ export function prepareMonitoringRcon(servers: string[]): Map<string, () => Prom
                     const operation = withOperationSource('Status monitor', () => queueHostOperation(instance.host_id, 'rcon.status-batch', {
                         instances: instances.map(value => ({ instanceId: value.id, generation: value.generation }))
                     }));
+                    // Allow delivery and receipt overhead beyond the agent's 20-second budget.
                     const receipt = await waitHostOperationStatus(operation.id, 25_000);
                     if (receipt.state !== 'completed') throw new Error(receipt.error ?? 'Host status batch did not complete.');
                     return receipt.result as { results: Array<{ instanceId: string; generation: number; reply: AgentRconReply }> };
@@ -211,8 +213,12 @@ export function prepareMonitoringRcon(servers: string[]): Map<string, () => Prom
                 const receipt = await batch.result;
                 const replies = receipt.results.filter(value => value.instanceId === instance.id && value.generation === instance.generation);
                 if (replies.length !== 1 || typeof replies[0].reply?.text !== 'string') throw new Error('Host status batch omitted a valid instance result.');
+                if (replies[0].reply.reason === 'deadline') throw new MonitoringSkippedError('Host status batch budget expired before this server was checked.');
                 return agentRconStatusResult(replies[0].reply, endpoint);
-            } catch (error) { return { ok: false as const, endpoint, reason: 'timeout' as const, detail: (error as Error).message }; }
+            } catch (error) {
+                if (error instanceof MonitoringSkippedError) throw error;
+                return { ok: false as const, endpoint, reason: 'timeout' as const, detail: (error as Error).message };
+            }
         };
         // Observe before dispatch, including servers still waiting for their
         // queue slot. Consume failures too, so a failed batch is never retried