import { config } from '../config.js'; import { db } from '../db/index.js'; import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js'; import { rconStatusResult } from '../integrations/rcon.js'; import { beginDeployment, deploymentStage, finishDeployment } from '../fleet/deployments.js'; // Komodo queues restart work. Don't mistake the still-running old container // for a completed restart when an API kick immediately follows the request. const HEALTH_POLL_MS = 15_000; const requestsInFlight = new Set(); interface RestartSlot { server: string; requested_at: string; checked_at: string | null; healthy_at: string | null; players: number | null; error: string | null; operation_id: string | null; } function slotFor(server: string): RestartSlot | undefined { return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined; } /** The queued restart may still expose the previous healthy container. */ export function restartSettling(server: string): boolean { const slot = slotFor(server); return requestsInFlight.has(server) || !!(slot && !slot.healthy_at && Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS); } export function rolloutWaitReason(server: string, operationId = ''): string | null { if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ? AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health')) OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) { return 'Waiting for the current provisioning operation'; } const slot = slotFor(server); if (slot?.healthy_at === null) { return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`; } const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number }; return n >= config.restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null; } export type RolloutRestartResult = | { kind: 'started' } | { kind: 'queued'; message: string } | { kind: 'failed'; message: string }; /** Reserve capacity before any async I/O; callers still check for players first. */ export async function restartInRollout(server: string, options: { operationId?: string; action?: 'start' | 'restart'; triggeredBy?: string } = {}): Promise { let operationId = options.operationId; const waitReason = db().transaction(() => { const reason = rolloutWaitReason(server, operationId); if (reason) return reason; operationId ??= beginDeployment(server, options.action ?? 'restart', options.triggeredBy ?? 'automation'); db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?) ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at, checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`) .run(server, new Date().toISOString(), operationId); deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container'); return null; }).immediate(); if (waitReason) return { kind: 'queued', message: waitReason }; requestsInFlight.add(server); db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server); try { const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server); if (result.ok) { db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(operationId); deploymentStage(operationId!, 'Waiting for container and RCON health'); return { kind: 'started' }; } // These failures cannot have queued a remote restart. Other failures // (especially a timeout) may have: retain capacity until recovery. if (result.kind === 'no-name' || result.kind === 'unconfigured') { db().prepare('DELETE FROM restart_rollout WHERE server = ?').run(server); } else { db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server); } finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` }); return { kind: 'failed', message: `komodo: ${result.message}` }; } catch (err) { const message = `komodo: ${(err as Error).message}`; db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server); finishDeployment(operationId!, { ok: false, message }); return { kind: 'failed', message }; } finally { // Start the grace period at completion, including slow/failed requests. db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ?') .run(new Date().toISOString(), server); requestsInFlight.delete(server); } } let refreshInFlight: Promise | null = null; /** Poll all occupied slots, even if their originating beta was already closed. */ export function refreshRestartRollout(): Promise { if (refreshInFlight) return refreshInFlight; refreshInFlight = refresh().finally(() => { refreshInFlight = null; }); return refreshInFlight; } async function refresh(): Promise { // Adopt work started before this feature was installed. INSERT OR IGNORE // preserves completed slots and never resets a newer restart's progress. db().prepare(`INSERT OR IGNORE INTO restart_rollout (server, requested_at) SELECT server, MAX(restarted_at) FROM deployment_servers WHERE restart_state IN ('pending','failed') AND restarted_at IS NOT NULL GROUP BY server`).run(); db().prepare(`INSERT OR IGNORE INTO restart_rollout (server, requested_at) SELECT bs.server, MAX(COALESCE(bs.restarted_at, bs.updated_at)) FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE b.state = 'running' AND bs.stage IN ('restarting','awaiting-health') GROUP BY bs.server`).run(); const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server') .all() as RestartSlot[]; for (const slot of slots) { if (requestsInFlight.has(slot.server)) continue; if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue; let error: string | null = null; let players: number | null = null; try { const health = await komodoInspectHealthy(slot.server); if (!health.ok) { error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`; } else { const rcon = await rconStatusResult(slot.server); if (!rcon.ok) error = `rcon not answering: ${rcon.reason}`; else players = rcon.status.players; } } catch (err) { error = `health check: ${(err as Error).message}`; } const now = new Date().toISOString(); db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ? WHERE server = ? AND requested_at = ? AND healthy_at IS NULL`) .run(now, error ? null : now, players, error, slot.server, slot.requested_at); if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'awaiting-health'").get(slot.operation_id)) { if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error); else { deploymentStage(slot.operation_id, 'Verified container and RCON health'); finishDeployment(slot.operation_id, { ok: true }); } } } } export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } { const slot = slotFor(server); if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 }; return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' }; }