Fix transient health checks and avoid unnecessary recovery

src/lib/server/provisioning/requests.ts

@@ -139,12 +139,21 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
     return db().transaction(() => {
         const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;
         if (blocker) return { ok: false as const, status: 409, error: blocker };
-        const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
+        const existing = db().prepare(`SELECT id, action, state, submitted_at, triggered_by FROM server_deployments WHERE server = ?
             AND server_deleted_at IS NULL
-            AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
+            AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as
+            { id: string; action: string; state: string; submitted_at: string | null; triggered_by: string } | undefined;
         if (existing) {
             if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
             db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
+            if (existing.state === 'waiting' && !existing.submitted_at && existing.triggered_by.startsWith('watchdog:')
+                && db().prepare(`SELECT 1 FROM provisioning_sources s JOIN watchdog_incidents i ON i.id = s.source_id
+                    WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health'`).get(existing.id)) {
+                // An operator explicitly requesting this queued action keeps
+                // that intent even if the original health condition clears.
+                db().prepare("UPDATE server_deployments SET triggered_by = 'ui' WHERE id = ?").run(existing.id);
+                recordDeploymentEvent(existing.id, 'Operator explicitly requested the queued health recovery action');
+            }
             return { ok: true as const, deploymentId: existing.id };
         }
         const row = getServer(server);

src/lib/server/provisioning/worker.ts

@@ -1,11 +1,12 @@
 import { maintenanceAllowed } from '../maintenance/policies.js';
+import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
 import { db } from '../db/index.js';
 import { komodoConfigured, rconConfigured } from '../config.js';
 import { getServer } from '../fleet/servers.js';
 import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
 import { processCreations, processDestructions, rebuildServer } from './instances.js';
 import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
-import { probePlayerEvidence } from '../fleet/status-evidence.js';
+import { latestGameSubmission, probePlayerEvidence } from '../fleet/status-evidence.js';
 import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js';
 import { cancelProvisioning } from './requests.js';
 import type { RuntimeAction } from '../../provisioning.js';
@@ -158,11 +159,28 @@ async function processOperation(operation: Operation): Promise<void> {
     }
     const reason = blocker(server, id, action);
     if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; }
+    const healthRecovery = watchdog && !!db().prepare(`SELECT 1 FROM provisioning_sources s
+        JOIN watchdog_incidents i ON i.id = s.source_id JOIN server_deployments d ON d.id = s.operation_id
+        WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health' AND d.triggered_by LIKE 'watchdog:%'`).get(id);
+    const epoch = latestGameSubmission(server);
     const health = await komodoInspectHealthy(server);
+    if (healthRecovery && epoch !== latestGameSubmission(server)) return;
+    if (healthRecovery && !health.ok && !['stopped','exited','created','dead','unhealthy','starting','restarting'].includes(health.state?.toLowerCase() ?? '')) {
+        deploymentStage(id, 'Waiting for current container health before automatic recovery', { phase: 'eligibility', checks: [],
+            blocker: { code: 'observation-unavailable', message: health.reason, automatic: true } });
+        return;
+    }
     if (action !== 'start' && !isStopped(health.state)) {
         const evidence = await probePlayerEvidence(server);
         if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
         const players = evidence.players;
+        if (healthRecovery && epoch !== latestGameSubmission(server)) return;
+        if (healthRecovery && health.ok && players !== null) {
+            // The fresh occupancy check also proves that the game recovered.
+            // Empty is not a reason to execute an obsolete health restart.
+            resolveRecoveredHealthOperation(id, players, evidence.source);
+            return;
+        }
         recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting',
             source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }],
             blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present',