import { fleetSettings } from '../settings.js'; import { db } from '../db/index.js'; import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js'; import { probeServerStatus } from '../fleet/status-evidence.js'; import { deploymentStage, finishDeployment } from './history.js'; import { applyStagedFilesForOperation } from '../fleet/staged-files.js'; import type { RuntimeAction } from '../../provisioning.js'; import type { ProvisioningBlocker } from '../../provisioning-detail.js'; import { recordProgress, type ProgressUpdate } from './progress.js'; import { nativeDeployment } from '../backends/native.js'; // Komodo queues restart work. Don't mistake the still-running old container // for a completed restart when an API kick immediately follows the request. const HEALTH_POLL_MS = 15_000; const requestsInFlight = new Map(); export const restartRequestInFlight = (server: string) => requestsInFlight.has(server); interface RestartSlot { server: string; requested_at: string; checked_at: string | null; healthy_at: string | null; players: number | null; error: string | null; operation_id: string | null; } function slotFor(server: string): RestartSlot | undefined { return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined; } export function restartRecoveryPending(server: string): boolean { return slotFor(server)?.healthy_at === null; } /** The queued restart may still expose the previous healthy container. */ export function restartSettling(server: string): boolean { const slot = slotFor(server); return requestsInFlight.has(server) || !!(slot && !slot.healthy_at && Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS); } export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null { return rolloutBlocker(server, operationId, action)?.message ?? null; } export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null { const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health')) OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined; if (conflict) { return { code: 'operation-conflict', message: 'Waiting for the current provisioning operation', automatic: true, href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' }; } const slot = slotFor(server); const ownHandoff = slot?.operation_id === operationId && !!db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS ( SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId); if (slot?.healthy_at === null && !ownHandoff) { return { code: 'previous-recovery', message: `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`, automatic: true, href: slot.operation_id ? `/provisioning/${encodeURIComponent(slot.operation_id)}` : null, label: 'Previous operation' }; } const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number }; return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= fleetSettings().restartConcurrency ? { code: 'restart-capacity', message: `Queued to ${action}; waiting for another server to recover`, automatic: true } : null; } export type RolloutRestartResult = | { kind: 'started' } | { kind: 'queued'; message: string } | { kind: 'failed'; message: string }; /** Reserve capacity before any async I/O; callers still check for players first. */ export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise { const operationId = options.operationId; const waitReason = db().transaction(() => { if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted'; const reason = rolloutWaitReason(server, operationId, options.action); if (reason) return reason; db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?) ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at, checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`) .run(server, new Date().toISOString(), operationId); recordProgress(operationId, { blocker: null, recovery: { state: 'held', message: 'Restart recovery capacity reserved.', operationId } }); return null; }).immediate(); if (waitReason) return { kind: 'queued', message: waitReason }; // Every gate, including rollout capacity, has cleared. No async work may // intervene between applying these files and submitting the restart. try { applyStagedFilesForOperation(operationId, server); } catch (err) { const message = `File activation failed: ${(err as Error).message}`; db().transaction(() => { db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId); recordProgress(operationId, { recovery: { state: 'released', message: 'Capacity released before any remote command: file activation failed.', operationId } }); finishDeployment(operationId, { ok: false, message }); })(); return { kind: 'failed', message }; } db().transaction(() => { db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId); deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container', { phase: 'container-action', blocker: null, checks: [], checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' }, command: { state: 'attempted', message: `${options.action === 'start' ? 'Start' : 'Restart'} submission attempted.` } }); })(); requestsInFlight.set(server, operationId); db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, rcon_failures = 0, last_reply_snippet = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server); try { const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server); if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' }; if (result.ok) { db().transaction(() => { db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId); deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)', { phase: 'verify-container', command: { state: 'accepted', message: 'Komodo accepted the request; recovery still requires verification.' }, checkpoint: { key: 'container-action', label: 'Container command accepted' }, blocker: { code: 'recovery-settling', message: 'Waiting for the recovery settling interval before checking health.', automatic: true } }); })(); return { kind: 'started' }; } // These failures cannot have queued a remote restart. Other failures // (especially a timeout) may have: retain capacity until recovery. db().transaction(() => { if (result.kind === 'no-name' || result.kind === 'unconfigured') { db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId); recordProgress(operationId, { command: { state: 'rejected', message: 'The command could not be submitted: ' + result.message }, recovery: { state: 'released', message: 'Capacity released because no remote command could be submitted.', operationId } }); } else { db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId); recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + result.message } }); } finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` }); })(); return { kind: 'failed', message: `komodo: ${result.message}` }; } catch (err) { if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' }; const message = `komodo: ${(err as Error).message}`; db().transaction(() => { db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId); recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + (err as Error).message } }); finishDeployment(operationId!, { ok: false, message }); })(); return { kind: 'failed', message }; } finally { // Start the grace period at completion, including slow/failed requests. db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ? AND operation_id = ?') .run(new Date().toISOString(), server, operationId); if (requestsInFlight.get(server) === operationId) requestsInFlight.delete(server); } } let refreshInFlight: Promise | null = null; /** Poll all occupied slots, even if their originating beta was already closed. */ export function refreshRestartRollout(): Promise { if (refreshInFlight) return refreshInFlight; refreshInFlight = refresh().finally(() => { refreshInFlight = null; }); return refreshInFlight; } async function refresh(): Promise { const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server') .all() as RestartSlot[]; for (const slot of slots) { if (slot.operation_id && nativeDeployment(slot.operation_id)) continue; if (requestsInFlight.has(slot.server)) continue; // Recovery capacity transferred to a rebuild remains reserved until // teardown. Its health is verified by the creation worker, never here. if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND d.server_deleted_at IS NULL AND ( (d.action = 'rebuild' AND d.state IN ('waiting','queued','running','awaiting-health')) OR (d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS ( SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')))`) .get(slot.operation_id)) continue; if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue; let error: string | null = null; let players: number | null = null; const checks: NonNullable = []; try { const health = await komodoInspectHealthy(slot.server); checks.push({ key: 'container', label: 'Container readiness', result: health.ok ? 'pass' : 'waiting', source: 'komodo', actual: health.state ?? null, checkedAt: new Date().toISOString(), message: health.ok ? null : health.reason }); if (!health.ok) { error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`; } else { const status = await probeServerStatus(slot.server, slot.requested_at); checks.push({ key: 'game', label: 'Game status', result: status.ok ? 'pass' : 'waiting', source: status.ok ? status.source : null, checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null, message: status.ok ? null : status.reason }); if (!status.ok) error = `rcon not answering and no fresh telemetry: ${status.reason}`; else players = status.players; } } catch (err) { error = `health check: ${(err as Error).message}`; checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', result: 'unknown', source: null, message: error }); } const now = new Date().toISOString(); db().transaction(() => { const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ? WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`) .run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id); if (!checked.changes) return; if (slot.operation_id) recordProgress(slot.operation_id, { phase: checks.some(check => check.key === 'game') ? 'verify-game' : 'verify-container', checks, observedAt: now, blocker: error ? { code: 'recovery-health', message: error, automatic: true } : null, recovery: { state: error ? 'held' : 'released', at: now, operationId: slot.operation_id, message: error ? 'Restart capacity remains reserved while recovery is checked.' : 'Container and game recovery verified; restart capacity released.' }, checkpoints: error ? [] : [{ key: 'verify-container', label: 'Container readiness verified', at: now }, { key: 'verify-game', label: 'Game status verified', at: now }] }); if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) { db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id); if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error); else { deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)'); finishDeployment(slot.operation_id, { ok: true }); } } })(); } } export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } { const slot = slotFor(server); if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 }; return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' }; }