import { db } from '../db/index.js'; import { komodoConfigured, rconConfigured } from '../config.js'; import { getServer } from '../fleet/servers.js'; import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js'; import { processCreations, processDestructions, rebuildServer } from './instances.js'; import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js'; import { probePlayerCount } from '../fleet/status-evidence.js'; import { refreshRestartRollout, restartInRollout, rolloutWaitReason } from './restarts.js'; import { cancelProvisioning } from './requests.js'; import type { RuntimeAction } from '../../provisioning.js'; const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']); export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? ''); interface Source { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string } const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?').all(id) as Source[]; const priority = (sources: Source[]) => sources.some(s => s.kind === 'watchdog') ? 2 : sources.some(s => s.kind === 'beta') ? 0 : sources.some(s => s.kind === 'campaign') ? 1 : sources.some(s => s.kind === 'maintenance') ? 3 : 2; /** Source work may hold or withdraw a request, but never executes it. */ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled' { const sources = sourcesFor(operation.id); if (!sources.length) return 'ready'; // Watchdog intent gates every linked source, including its drift campaign. for (const source of sources.filter(source => source.kind === 'watchdog')) { if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(source.source_id, operation.server) || getServer(operation.server)?.desired_state === 'stopped') return 'cancelled'; if (db().prepare("SELECT 1 FROM settings WHERE key = 'watchdog_enabled' AND value = '0'").get()) return 'held'; } let held = false; for (const source of sources) { if (source.kind === 'campaign') { const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ?') .get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined; if (row?.provisioning_id !== operation.id || !row.files_changed) continue; if (['pending','failed'].includes(row.restart_state)) return 'ready'; if (['held','manual'].includes(row.restart_state)) held = true; } else if (source.kind === 'beta') { const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ?`) .get(source.source_id, operation.server) as { state: string; stage: string; provisioning_id: string; outcome: string | null } | undefined; if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue; if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue; if (!row.outcome || ['active','passed'].includes(row.outcome)) return 'ready'; if (row.outcome === 'paused') held = true; } else if (source.kind === 'maintenance') { const row = db().prepare('SELECT state, provisioning_id FROM maintenance_jobs WHERE id = ?').get(source.source_id) as { state: string; provisioning_id: string } | undefined; if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) return 'ready'; } else return 'ready'; } return held ? 'held' : 'cancelled'; } function blocker(server: string, id: string): string | null { const sources = sourcesFor(id); const rank = priority(sources); const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND state IN ('queued','waiting','running','awaiting-health')`).all(server, id) as { id: string; state: string; action: string; rowid: number }[]; const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { rowid: number }; if (others.some(other => other.state !== 'waiting' || other.action === 'destroy' || priority(sourcesFor(other.id)) < rank || (priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid))) { return 'Waiting for another provisioning operation on this server'; } // Beta activation must not wait on the campaign rows that the beta itself // holds. Campaign/maintenance callers likewise never block on their own job. if (rank > 0 && db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.server = ? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server'; if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ? AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS ( SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health'))) AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id) LIMIT 1`).get(server, id)) return 'Waiting for campaign work on this server'; return rolloutWaitReason(server, id); } interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string } let ticking: Promise | null = null; export function processProvisioning(): Promise { if (ticking) return ticking; ticking = run().finally(() => { ticking = null; }); return ticking; } export const processLifecycle = processProvisioning; async function run(): Promise { await processCreations(); await processDestructions(); await refreshRestartRollout(); const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments WHERE action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health')) ORDER BY started_at, rowid`).all() as Operation[]; for (const operation of operations) { try { await processOperation(operation); } catch (err) { // Observation failures are retried; remote mutations are never replayed. deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message); } } } async function processOperation(operation: Operation): Promise { const { id, server, action } = operation; // An explicit cleanup resume follows confirmed remote deletion. Complete // that durable checkpoint even if the incident now needs attention or the // Watchdog is paused; never strand the reserved replacement identity. if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; } if (operation.state === 'waiting') { const eligibility = sourceEligibility(operation); if (eligibility === 'cancelled') { cancelProvisioning(id, 'Requester cancelled or replaced this operation'); return; } if (eligibility === 'held') { deploymentStage(id, 'Waiting for the requester to resume'); return; } } if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; } const watchdog = sourcesFor(id).some(source => source.kind === 'watchdog'); if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; } if (operation.state === 'awaiting-health') { const health = await komodoInspectHealthy(server); if (isStopped(health.state)) { db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server); deploymentStage(id, 'Verified container stopped'); finishDeployment(id, { ok: true }); } else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : '')); return; } const reason = blocker(server, id); if (reason) { deploymentStage(id, reason); return; } const health = await komodoInspectHealthy(server); if (action !== 'start' && !isStopped(health.state)) { const players = await probePlayerCount(server); db().prepare('UPDATE server_deployments SET players = ?, checked_at = ? WHERE id = ?').run(players, new Date().toISOString(), id); if (players !== 0) { deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`); return; } } // Requests can arrive during the probes. Recheck before claiming the action. if (blocker(server, id) || sourceEligibility(operation) !== 'ready') return; if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'waiting'").get(id)) return; if (action === 'rebuild') { await rebuildServer(id); return; } const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id); if (!claim.changes) return; if (action === 'start' || action === 'restart') { const result = await restartInRollout(server, { operationId: id, action }); if (result.kind === 'queued') { db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); deploymentStage(id, result.message); } return; } db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id); deploymentStage(id, 'Stopping container'); // Persist intent first; a lost reply must not issue another stop later. try { const result = await komodoRuntimeAction(server, 'stop'); if (!result.ok) { finishDeployment(id, { ok: false, message: result.message }); return; } db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id); deploymentStage(id, 'Waiting for container to stop'); } catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); } }