Fix transient health checks and avoid unnecessary recovery

src/lib/server/provisioning/worker.ts

@@ -1,11 +1,12 @@
 import { maintenanceAllowed } from '../maintenance/policies.js';
+import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
 import { db } from '../db/index.js';
 import { komodoConfigured, rconConfigured } from '../config.js';
 import { getServer } from '../fleet/servers.js';
 import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
 import { processCreations, processDestructions, rebuildServer } from './instances.js';
 import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
-import { probePlayerEvidence } from '../fleet/status-evidence.js';
+import { latestGameSubmission, probePlayerEvidence } from '../fleet/status-evidence.js';
 import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js';
 import { cancelProvisioning } from './requests.js';
 import type { RuntimeAction } from '../../provisioning.js';
@@ -158,11 +159,28 @@ async function processOperation(operation: Operation): Promise<void> {
     }
     const reason = blocker(server, id, action);
     if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; }
+    const healthRecovery = watchdog && !!db().prepare(`SELECT 1 FROM provisioning_sources s
+        JOIN watchdog_incidents i ON i.id = s.source_id JOIN server_deployments d ON d.id = s.operation_id
+        WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health' AND d.triggered_by LIKE 'watchdog:%'`).get(id);
+    const epoch = latestGameSubmission(server);
     const health = await komodoInspectHealthy(server);
+    if (healthRecovery && epoch !== latestGameSubmission(server)) return;
+    if (healthRecovery && !health.ok && !['stopped','exited','created','dead','unhealthy','starting','restarting'].includes(health.state?.toLowerCase() ?? '')) {
+        deploymentStage(id, 'Waiting for current container health before automatic recovery', { phase: 'eligibility', checks: [],
+            blocker: { code: 'observation-unavailable', message: health.reason, automatic: true } });
+        return;
+    }
     if (action !== 'start' && !isStopped(health.state)) {
         const evidence = await probePlayerEvidence(server);
         if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
         const players = evidence.players;
+        if (healthRecovery && epoch !== latestGameSubmission(server)) return;
+        if (healthRecovery && health.ok && players !== null) {
+            // The fresh occupancy check also proves that the game recovered.
+            // Empty is not a reason to execute an obsolete health restart.
+            resolveRecoveredHealthOperation(id, players, evidence.source);
+            return;
+        }
         recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting',
             source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }],
             blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present',