import { db } from '../db/index.js'; import { komodoConfigured, rconConfigured } from '../config.js'; import { getServer } from '../fleet/servers.js'; import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js'; import { queueRebuildServer } from './instances.js'; import { restartRequestInFlight } from './restarts.js'; import { runtimeActions, type RuntimeAction } from '../../provisioning.js'; export interface LifecycleOperation { id: string; server: string; action: string; state: string; stage: string; submitted_at: string | null; finished_at: string | null; checked_at: string | null; players: number | null; error: string | null; started_at: string; } export interface RestartSource { kind: 'campaign' | 'beta' | 'maintenance'; id: string } export function provisioningOperation(id: string): LifecycleOperation | undefined { return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id) as LifecycleOperation | undefined; } const sourceTables = { campaign: { table: 'deployment_servers', key: 'deployment_id' }, beta: { table: 'beta_servers', key: 'beta_id' }, maintenance: { table: 'maintenance_jobs', key: 'id' } } as const; /** Submit once per caller attempt; a lost response or terminal failure never replays a restart. */ export function requestRestart(server: string, source: RestartSource): LifecycleOperation { return db().transaction(() => { const { table, key } = sourceTables[source.kind]; const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ?`) .get(source.id, server) as { provisioning_id: string | null; restarted_at: string | null } | undefined; if (!row) throw new Error('Restart source no longer exists'); if (row.provisioning_id) { const operation = provisioningOperation(row.provisioning_id); if (!operation) throw new Error('Provisioning job is missing; inspect the server before retrying'); if (row.restarted_at && !operation.submitted_at && operation.state === 'waiting') { adoptSubmittedRestart(operation.id, server, row.restarted_at); return provisioningOperation(operation.id)!; } return operation; } let id: string | undefined; if (row.restarted_at) { // The previous release already recorded some submissions in history. // Bind to that exact interval, never to a newer restart of the server. id = (db().prepare(`SELECT d.id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id WHERE r.server = ? AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`) .get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id; if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id); } // Only not-yet-submitted campaign changes can share an activation. A later // file deployment must never be credited to an already running restart. if (source.kind === 'campaign' && !row.restarted_at) { id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign') AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign') ORDER BY d.rowid LIMIT 1`).get(server) as { id: string } | undefined)?.id; } if (!id) { id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); deploymentStage(id, 'Waiting to verify the server is empty'); if (row.restarted_at) adoptSubmittedRestart(id, server, row.restarted_at); } db().prepare('INSERT OR IGNORE INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, ?, ?, ?)') .run(id, source.kind, source.id, server); db().prepare(`UPDATE ${table} SET provisioning_id = ? WHERE ${key} = ? AND server = ?`).run(id, source.id, server); return provisioningOperation(id)!; })(); } /** Upgrade observation only: existing restart intent is never resubmitted. */ function adoptSubmittedRestart(id: string, server: string, submittedAt: string): void { const slot = db().prepare('SELECT operation_id, requested_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as { operation_id: string | null; requested_at: string; healthy_at: string | null; players: number | null } | undefined; db().prepare("UPDATE server_deployments SET state = 'awaiting-health', submitted_at = ? WHERE id = ?").run(submittedAt, id); if (slot && ((slot.operation_id && slot.requested_at >= submittedAt) || (slot.requested_at < submittedAt && !slot.healthy_at))) { db().prepare("UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE id = ?") .run(new Date().toISOString(), 'Existing restart cannot be correlated safely. Inspect the server before retrying.', id); return; } if (slot && slot.requested_at >= submittedAt) { db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ?').run(id, server); } else { db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?) ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at, operation_id = excluded.operation_id, checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`).run(server, submittedAt, id); } if (slot?.healthy_at && slot.requested_at >= submittedAt) { db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ?, players = ? WHERE id = ?").run(slot.healthy_at, slot.players, id); } deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again'); } export function cancelProvisioning(id: string, reason = 'Cancelled by requester'): boolean { return db().transaction(() => { const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL`) .run(new Date().toISOString(), reason, id); if (result.changes) recordDeploymentEvent(id, reason); return result.changes > 0; })(); } /** Explicit caller retry starts a new job and keeps the old job as history. */ export function releaseRestartRequest(server: string, source: RestartSource): void { const { table, key } = sourceTables[source.kind]; const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ?`).get(source.id, server) as { provisioning_id: string | null } | undefined; if (row?.provisioning_id) cancelProvisioning(row.provisioning_id, 'Superseded by a fresh caller attempt'); db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ?`).run(source.id, server); } export function requestLifecycle(server: string, action: RuntimeAction) { if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' }; if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' }; return db().transaction(() => { const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ? AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined; if (existing) { if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' }; db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server); return { ok: true as const, deploymentId: existing.id }; } const row = getServer(server); if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' }; if (action === 'rebuild') { const result = queueRebuildServer(server); if (result.ok) db().prepare("UPDATE servers SET desired_state = 'running' WHERE name = ?").run(server); return result.ok ? { ok: true as const, deploymentId: result.deploymentId } : { ok: false as const, status: 400, error: result.message }; } const id = beginDeployment(server, action); db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty'); return { ok: true as const, deploymentId: id }; })(); } type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string }; function watchdogRequestGuard(server: string, incidentId: string): Extract | null { const incident = db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(incidentId, server); if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' }; if (db().prepare("SELECT 1 FROM settings WHERE key = 'watchdog_enabled' AND value = '0'").get()) { return { ok: false, status: 409, error: 'Watchdog is paused.' }; } const row = getServer(server); if (!row || row.destroyed_at) return { ok: false, status: 404, error: 'Server not found.' }; if (row.desired_state === 'stopped') return { ok: false, status: 409, error: 'This server is intentionally stopped.' }; if (!komodoConfigured()) return { ok: false, status: 503, error: 'Configure Komodo before provisioning.' }; return null; } function existingWatchdogAction(incidentId: string, action: 'restart' | 'rebuild'): string | undefined { return (db().prepare(`SELECT a.operation_id FROM watchdog_actions a JOIN watchdog_incidents i ON i.id = a.incident_id WHERE a.incident_id = ? AND a.action = ? AND a.cycle = i.cycle`).get(incidentId, action) as { operation_id: string } | undefined)?.operation_id; } function queueWatchdogAction(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult { let id: string; if (action === 'rebuild') { const result = queueRebuildServer(server, `watchdog:${incidentId}`); if (!result.ok) return { ok: false, status: 400, error: result.message }; id = result.deploymentId; } else { id = beginDeployment(server, action, `watchdog:${incidentId}`); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); deploymentStage(id, 'Waiting to verify the server is empty'); } db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'watchdog', ?, ?)").run(id, incidentId, server); return { ok: true, deploymentId: id }; } /** Watchdog submits durable work; Provisioning remains the only execution owner. */ export function requestWatchdogLifecycle(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult { return db().transaction(() => { const previous = existingWatchdogAction(incidentId, action); if (previous) return { ok: true as const, deploymentId: previous }; const error = watchdogRequestGuard(server, incidentId); if (error) return error; const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog' WHERE d.server = ? AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`) .get(server) as { id: string; action: string; source_id: string | null } | undefined; if (pending) return pending.source_id === incidentId && pending.action === action ? { ok: true as const, deploymentId: pending.id } : { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' }; const next = queueWatchdogAction(server, action, incidentId); if (!next.ok) return next; // An operator retry starts a new incident cycle. Keep the previous // failed attempt's capacity, but let that new attempt use it instead // of waiting forever behind its own incident's terminal operation. const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id JOIN watchdog_actions a ON a.operation_id = d.id AND a.incident_id = ? JOIN watchdog_incidents i ON i.id = a.incident_id WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled') AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined; if (retained && !restartRequestInFlight(server)) { db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL') .run(next.deploymentId, server, retained.operation_id); recordDeploymentEvent(retained.operation_id, 'Recovery capacity handed to explicit Watchdog retry ' + next.deploymentId); recordDeploymentEvent(next.deploymentId, 'Retained recovery capacity from prior incident attempt ' + retained.operation_id); } return next; })(); } /** Atomically reserve a replacement before handing it the failed restart's capacity. */ export function escalateWatchdogRestart(incidentId: string, operationId: string, recoveryTimeoutMs = 300_000): LifecycleRequestResult { return db().transaction(() => { const previous = existingWatchdogAction(incidentId, 'rebuild'); if (previous) return { ok: true as const, deploymentId: previous }; const operation = provisioningOperation(operationId); if (!operation || operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state) || !db().prepare("SELECT 1 FROM provisioning_sources WHERE operation_id = ? AND kind = 'watchdog' AND source_id = ?").get(operationId, incidentId)) { return { ok: false as const, status: 409, error: 'The restart is not eligible for Watchdog escalation.' }; } const error = watchdogRequestGuard(operation.server, incidentId); if (error) return error; const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog' WHERE d.server = ? AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`) .get(operation.server, operationId) as { id: string; action: string; source_id: string | null } | undefined; if (pending) return pending.action === 'rebuild' && pending.source_id === incidentId ? { ok: true as const, deploymentId: pending.id } : { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' }; const slot = db().prepare('SELECT operation_id, requested_at, healthy_at FROM restart_rollout WHERE server = ?').get(operation.server) as { operation_id: string | null; requested_at: string; healthy_at: string | null } | undefined; const anchor = Date.parse(slot?.requested_at ?? operation.submitted_at ?? operation.started_at); if (restartRequestInFlight(operation.server) || (slot && (slot.operation_id !== operationId || slot.healthy_at)) || !Number.isFinite(anchor) || Date.now() - anchor < Math.max(300_000, recoveryTimeoutMs)) { return { ok: false as const, status: 409, error: 'Waiting for the submitted restart to finish its recovery window.' }; } const replacement = queueWatchdogAction(operation.server, 'rebuild', incidentId); if (!replacement.ok) return replacement; const reason = 'Watchdog restart recovery timed out; handed recovery to rebuild ' + replacement.deploymentId; if (operation.state === 'awaiting-health') finishDeployment(operationId, { ok: false, message: reason }); else recordDeploymentEvent(operationId, reason); if (slot) db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL') .run(replacement.deploymentId, reason, operation.server, operationId); recordDeploymentEvent(replacement.deploymentId, 'Recovery handed off from restart ' + operationId + '; waiting for a fresh empty or stopped server'); return replacement; })(); }