Share recent health observations between background monitors

src/lib/server/maintenance/recovery.ts

@@ -4,8 +4,7 @@ import { komodoConfigured } from '../config.js';
 import { nativeInstance } from '../backends/native.js';
 import { getServer, listServers } from '../fleet/servers.js';
 import { hasActiveProvisioning } from '../fleet/provisioning.js';
-import { komodoInspectHealthy } from '../integrations/komodo.js';
-import { rconStatusResult } from '../integrations/rcon.js';
+import { inspectForMonitoring, rconForMonitoring, MonitoringChangedError } from '../fleet/health-observations.js';
 import { latestGameSubmission, telemetryEvidence, telemetryReport } from '../fleet/status-evidence.js';
 import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '../fleet/drift.js';
 import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
@@ -223,17 +222,27 @@ async function reconcileActions(): Promise<void> {
 }
 
 interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; players?: number; source?: 'rcon' | 'telemetry' }
-async function observe(server: string, operationId?: string): Promise<Observation | null> {
+async function observe(server: string, operationId?: string, fresh = false): Promise<Observation | null> {
     const epoch = latestGameSubmission(server);
     const suppressed = () => (operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server))
         || protectedWork(server) || getServer(server)?.desired_state !== 'running' || epoch !== latestGameSubmission(server);
-    const health = await komodoInspectHealthy(server);
+    const inspected = await inspectForMonitoring(server, fresh || !!operationId).catch(error => {
+        if (error instanceof MonitoringChangedError) return null;
+        throw error;
+    });
+    if (!inspected) return null;
+    const health = inspected.result;
     if (suppressed()) return null;
     const state = health.state?.toLowerCase() ?? null;
     if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
     const telemetry = telemetryEvidence(server);
     if (telemetry) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
-    const rcon = await rconStatusResult(server);
+    const probed = await rconForMonitoring(server, fresh || !!operationId).catch(error => {
+        if (error instanceof MonitoringChangedError) return null;
+        throw error;
+    });
+    if (!probed) return null;
+    const rcon = probed.result;
     if (suppressed()) return null;
     const latest = telemetryEvidence(server);
     if (rcon.ok || latest) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state,
@@ -245,7 +254,7 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
     const row = getServer(server);
     if (!row || row.desired_state === 'stopped') return;
     if (scanSuppressed(server) || protectedWork(server)) return;
-    const observation = await observe(server);
+    const observation = await observe(server, undefined, forceScan);
     // Probes can overlap an API request; recheck ownership before queuing anything.
     if (!observation || scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
     if (!scanSuppressed(server, 'health') && observation.healthy) {