import { maintenanceAllowed } from '../maintenance/policies.js'; import { db } from '../db/index.js'; import { komodoConfigured, rconConfigured } from '../config.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js'; import { queueRebuildServer } from './instances.js'; import { restartRequestInFlight } from './restarts.js'; import { runtimeActions, type RuntimeAction } from '../../provisioning.js'; import { recordProgress } from './progress.js'; import { nativeInstance, unplacedNativeCreation } from '../backends/native.js'; export interface LifecycleOperation { id: string; server: string; action: string; state: string; stage: string; submitted_at: string | null; finished_at: string | null; checked_at: string | null; players: number | null; error: string | null; started_at: string; } export interface RestartSource { kind: 'campaign' | 'beta' | 'maintenance'; id: string } export function provisioningOperation(id: string): LifecycleOperation | undefined { return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id) as LifecycleOperation | undefined; } const sourceTables = { campaign: { table: 'deployment_servers', key: 'deployment_id' }, beta: { table: 'beta_servers', key: 'beta_id' }, maintenance: { table: 'maintenance_jobs', key: 'id' } } as const; /** Submit once per caller attempt; a lost response or terminal failure never replays a restart. */ export function requestRestart(server: string, source: RestartSource): LifecycleOperation { return db().transaction(() => { const { table, key } = sourceTables[source.kind]; const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`) .get(source.id, server) as { provisioning_id: string | null; restarted_at: string | null } | undefined; if (!row) throw new Error('Restart source no longer exists'); if (row.provisioning_id) { const operation = provisioningOperation(row.provisioning_id); if (!operation) throw new Error('Provisioning job is missing; inspect the server before retrying'); if (row.restarted_at && !operation.submitted_at && operation.state === 'waiting') { adoptSubmittedRestart(operation.id, server, row.restarted_at); return provisioningOperation(operation.id)!; } return operation; } let id: string | undefined; if (row.restarted_at) { // The previous release already recorded some submissions in history. // Bind to that exact interval, never to a newer restart of the server. id = (db().prepare(`SELECT d.id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id WHERE r.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`) .get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id; if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id); } // Only not-yet-submitted campaign changes can share an activation. A later // file deployment must never be credited to an already running restart. // Restorations need their own activation. An older campaign can have // stale file preconditions after a beta; its failure must not strand // the rollback that would restore the server. const rollback = source.kind === 'campaign' && !!db().prepare("SELECT 1 FROM deployments WHERE id = ? AND action = 'rollback'").get(source.id); if (source.kind === 'campaign' && !row.restarted_at && !rollback && !nativeInstance(server)) { id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.stage_operation_id IS NOT NULL) AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign') AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign') AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN deployments c ON c.id = s.source_id WHERE s.operation_id = d.id AND s.kind = 'campaign' AND c.action = 'rollback') ORDER BY d.rowid LIMIT 1`).get(server) as { id: string } | undefined)?.id; } if (!id) { id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null, blocker: { code: 'eligibility', message: 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } }); if (row.restarted_at) adoptSubmittedRestart(id, server, row.restarted_at); } db().prepare('INSERT OR IGNORE INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, ?, ?, ?)') .run(id, source.kind, source.id, server); db().prepare(`UPDATE ${table} SET provisioning_id = ? WHERE ${key} = ? AND server = ?`).run(id, source.id, server); return provisioningOperation(id)!; })(); } /** Upgrade observation only: existing restart intent is never resubmitted. */ function adoptSubmittedRestart(id: string, server: string, submittedAt: string): void { const slot = db().prepare('SELECT operation_id, requested_at, checked_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as { operation_id: string | null; requested_at: string; checked_at: string | null; healthy_at: string | null; players: number | null } | undefined; db().prepare("UPDATE server_deployments SET state = 'awaiting-health', submitted_at = ? WHERE id = ?").run(submittedAt, id); recordProgress(id, { command: { state: 'attempted', at: submittedAt, message: 'Adopted a previously recorded submission; no command is replayed.' } }); if (slot && ((slot.operation_id && slot.requested_at >= submittedAt) || (slot.requested_at < submittedAt && !slot.healthy_at))) { const finishedAt = new Date().toISOString(); const error = 'Existing restart cannot be correlated safely. Inspect the server before retrying.'; db().prepare("UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE id = ?") .run(finishedAt, error, id); recordProgress(id, { recovery: { state: 'unknown', message: 'Existing restart recovery could not be correlated with this operation.', operationId: id }, observedAt: null, event: { kind: 'outcome', message: error, details: { state: 'interrupted', finishedAt, error } } }); return; } if (slot && slot.requested_at >= submittedAt) { db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ?').run(id, server); } else { db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?) ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at, operation_id = excluded.operation_id, checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`).run(server, submittedAt, id); } if (slot?.healthy_at && slot.requested_at >= submittedAt) { db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ?, players = ? WHERE id = ?").run(slot.healthy_at, slot.players, id); recordProgress(id, { observedAt: slot.healthy_at, event: { kind: 'outcome', message: 'Adopted recorded recovery completion.', details: { state: 'done', finishedAt: slot.healthy_at, error: null, warnings: [] } } }); } deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again', { phase: 'verify-container', checks: [], blocker: null, observedAt: slot?.checked_at ?? null, recovery: { state: slot?.healthy_at && slot.requested_at >= submittedAt ? 'released' : 'held', at: slot?.healthy_at && slot.requested_at >= submittedAt ? slot.healthy_at : undefined, message: slot?.healthy_at && slot.requested_at >= submittedAt ? 'Adopted recorded recovery completion.' : 'Observing the previously submitted restart without replaying it.', operationId: id } }); } export function cancelProvisioning(id: string, reason = 'Cancelled by requester'): boolean { return db().transaction(() => { const finishedAt = new Date().toISOString(); const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=server_deployments.id AND m.remove_operation_id IS NOT NULL)`) .run(finishedAt, reason, id); if (result.changes) recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome', message: reason, details: { state: 'cancelled', finishedAt, error: null, warnings: [] } } }); return result.changes > 0; })(); } /** Explicit caller retry starts a new job and keeps the old job as history. */ export function releaseRestartRequest(server: string, source: RestartSource): void { const { table, key } = sourceTables[source.kind]; const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`).get(source.id, server) as { provisioning_id: string | null } | undefined; if (row?.provisioning_id) cancelProvisioning(row.provisioning_id, 'Superseded by a fresh caller attempt'); db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`).run(source.id, server); } export function requestLifecycle(server: string, action: RuntimeAction) { if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' }; if (!nativeInstance(server) && !unplacedNativeCreation(server) && (!komodoConfigured() || !rconConfigured())) return { ok: false as const, status: 503, error: 'Configure a host and RCON before provisioning.' }; return db().transaction(() => { const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null; if (blocker) return { ok: false as const, status: 409, error: blocker }; const existing = db().prepare(`SELECT id, action, state, submitted_at, triggered_by FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string; state: string; submitted_at: string | null; triggered_by: string } | undefined; if (existing) { if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' }; db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server); if (existing.state === 'waiting' && !existing.submitted_at && existing.triggered_by.startsWith('watchdog:') && db().prepare(`SELECT 1 FROM provisioning_sources s JOIN watchdog_incidents i ON i.id = s.source_id WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health'`).get(existing.id)) { // An operator explicitly requesting this queued action keeps // that intent even if the original health condition clears. db().prepare("UPDATE server_deployments SET triggered_by = 'ui' WHERE id = ?").run(existing.id); recordDeploymentEvent(existing.id, 'Operator explicitly requested the queued health recovery action'); } return { ok: true as const, deploymentId: existing.id }; } const row = getServer(server); if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' }; if (action === 'rebuild') { const result = queueRebuildServer(server); if (result.ok) db().prepare("UPDATE servers SET desired_state = 'running' WHERE name = ?").run(server); return result.ok ? { ok: true as const, deploymentId: result.deploymentId } : { ok: false as const, status: result.kind === 'conflict' ? 409 : 400, error: result.message }; } const id = beginDeployment(server, action); db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null, blocker: { code: 'eligibility', message: action === 'start' ? 'Waiting for start eligibility and recovery capacity.' : 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } }); return { ok: true as const, deploymentId: id }; })(); } type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string }; function watchdogRequestGuard(server: string, incidentId: string): Extract | null { const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined; if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' }; if (!maintenanceAllowed(incident.kind)) { return { ok: false, status: 409, error: 'This maintenance policy is paused.' }; } const row = getServer(server); if (!row || row.destroyed_at) return { ok: false, status: 404, error: 'Server not found.' }; if (row.desired_state === 'stopped') return { ok: false, status: 409, error: 'This server is intentionally stopped.' }; if (!nativeInstance(server) && !komodoConfigured()) return { ok: false, status: 503, error: 'Configure a host before provisioning.' }; return null; } function existingWatchdogAction(incidentId: string, action: 'restart' | 'rebuild'): string | undefined { return (db().prepare(`SELECT a.operation_id FROM watchdog_actions a JOIN watchdog_incidents i ON i.id = a.incident_id WHERE a.incident_id = ? AND a.action = ? AND a.cycle = i.cycle`).get(incidentId, action) as { operation_id: string } | undefined)?.operation_id; } function queueWatchdogAction(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult { let id: string; if (action === 'rebuild') { const result = queueRebuildServer(server, `watchdog:${incidentId}`); if (!result.ok) return { ok: false, status: result.kind === 'conflict' ? 409 : 400, error: result.message }; id = result.deploymentId; } else { id = beginDeployment(server, action, `watchdog:${incidentId}`); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null, blocker: { code: 'eligibility', message: 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } }); } db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'watchdog', ?, ?)").run(id, incidentId, server); return { ok: true, deploymentId: id }; } /** Watchdog submits durable work; Provisioning remains the only execution owner. */ export function requestWatchdogLifecycle(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult { return db().transaction(() => { const previous = existingWatchdogAction(incidentId, action); if (previous) return { ok: true as const, deploymentId: previous }; const error = watchdogRequestGuard(server, incidentId); if (error) return error; const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog' WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`) .get(server) as { id: string; action: string; source_id: string | null } | undefined; if (pending) return pending.source_id === incidentId && pending.action === action ? { ok: true as const, deploymentId: pending.id } : { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' }; const next = queueWatchdogAction(server, action, incidentId); if (!next.ok) return next; // An operator retry starts a new incident cycle. Keep the previous // failed attempt's capacity, but let that new attempt use it instead // of waiting forever behind its own incident's terminal operation. const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id JOIN watchdog_actions a ON a.operation_id = d.id AND a.incident_id = ? JOIN watchdog_incidents i ON i.id = a.incident_id WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled') AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined; if (retained && !restartRequestInFlight(server)) { const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL') .run(next.deploymentId, server, retained.operation_id); if (transferred.changes) { recordProgress(retained.operation_id, { recovery: { state: 'transferred', operationId: next.deploymentId, message: 'Recovery capacity transferred to the explicit new incident attempt.' } }); recordProgress(next.deploymentId, { recovery: { state: 'held', operationId: next.deploymentId, message: 'Recovery capacity retained from the prior incident attempt.' } }); } recordDeploymentEvent(retained.operation_id, 'Recovery capacity handed to explicit Watchdog retry ' + next.deploymentId); recordDeploymentEvent(next.deploymentId, 'Retained recovery capacity from prior incident attempt ' + retained.operation_id); } return next; })(); } /** Atomically reserve a replacement before handing it the failed restart's capacity. */ export function escalateWatchdogRestart(incidentId: string, operationId: string, recoveryTimeoutMs = 300_000): LifecycleRequestResult { return db().transaction(() => { const previous = existingWatchdogAction(incidentId, 'rebuild'); if (previous) return { ok: true as const, deploymentId: previous }; const operation = provisioningOperation(operationId); if (!operation || operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state) || !db().prepare("SELECT 1 FROM provisioning_sources WHERE operation_id = ? AND kind = 'watchdog' AND source_id = ?").get(operationId, incidentId)) { return { ok: false as const, status: 409, error: 'The restart is not eligible for Watchdog escalation.' }; } const error = watchdogRequestGuard(operation.server, incidentId); if (error) return error; const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog' WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`) .get(operation.server, operationId) as { id: string; action: string; source_id: string | null } | undefined; if (pending) return pending.action === 'rebuild' && pending.source_id === incidentId ? { ok: true as const, deploymentId: pending.id } : { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' }; const slot = db().prepare('SELECT operation_id, requested_at, healthy_at FROM restart_rollout WHERE server = ?').get(operation.server) as { operation_id: string | null; requested_at: string; healthy_at: string | null } | undefined; const anchor = Date.parse(slot?.requested_at ?? operation.submitted_at ?? operation.started_at); if (restartRequestInFlight(operation.server) || (slot && (slot.operation_id !== operationId || slot.healthy_at)) || !Number.isFinite(anchor) || Date.now() - anchor < Math.max(300_000, recoveryTimeoutMs)) { return { ok: false as const, status: 409, error: 'Waiting for the submitted restart to finish its recovery window.' }; } const replacement = queueWatchdogAction(operation.server, 'rebuild', incidentId); if (!replacement.ok) return replacement; const reason = 'Watchdog restart recovery timed out; handed recovery to rebuild ' + replacement.deploymentId; if (operation.state === 'awaiting-health') finishDeployment(operationId, { ok: false, message: reason }); else recordDeploymentEvent(operationId, reason); if (slot) { const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL') .run(replacement.deploymentId, reason, operation.server, operationId); if (transferred.changes) { recordProgress(operationId, { recovery: { state: 'transferred', operationId: replacement.deploymentId, message: 'Recovery capacity transferred to the rebuild operation.' } }); recordProgress(replacement.deploymentId, { recovery: { state: 'held', operationId: replacement.deploymentId, message: 'Recovery capacity retained from the previous restart.' } }); } } recordDeploymentEvent(replacement.deploymentId, 'Recovery handed off from restart ' + operationId + '; waiting for a fresh empty or stopped server'); return replacement; })(); }