import { fleetSettings } from '../settings.js'; import { db } from '../db/index.js'; import type { RuntimeAction } from '../../provisioning.js'; import type { ProvisioningBlocker } from '../../provisioning-detail.js'; const HEALTH_POLL_MS = 15_000; interface RestartSlot { server: string; requested_at: string; checked_at: string | null; healthy_at: string | null; players: number | null; error: string | null; operation_id: string | null; } function slotFor(server: string): RestartSlot | undefined { return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined; } export function restartRecoveryPending(server: string): boolean { return slotFor(server)?.healthy_at === null; } type SettlingSlot = Pick; interface SettlingOperation { id: string; action: string; state: string } /** Submitted lifecycle work stays Starting until verification ends, including slow restarts. */ export function recoverySettling(slot: SettlingSlot | undefined, operation?: SettlingOperation | null): boolean { if (!slot || slot.healthy_at) return false; if (slot.operation_id) return operation?.id === slot.operation_id && ['start', 'restart', 'rebuild'].includes(operation.action) && ['running', 'awaiting-health'].includes(operation.state); // Legacy recovery slots have no operation whose verification can be followed. return Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS; } export function restartSettling(server: string): boolean { const slot = slotFor(server); const operation = slot?.operation_id ? db().prepare(`SELECT id, action, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as SettlingOperation | undefined : undefined; return recoverySettling(slot, operation); } export function rolloutWaitReason(server: string, operationId = '', action: RuntimeAction = 'restart'): string | null { return rolloutBlocker(server, operationId, action)?.message ?? null; } export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null { const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health')) OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined; if (conflict) { return { code: 'operation-conflict', message: 'Waiting for the current provisioning operation', automatic: true, href: `/provisioning/${encodeURIComponent(conflict.id)}`, label: 'Blocking operation' }; } const slot = slotFor(server); const ownHandoff = slot?.operation_id === operationId && !!db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS ( SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId); if (slot?.healthy_at === null && !ownHandoff) { return { code: 'previous-recovery', message: `Waiting for this server to recover from its previous operation${slot.error ? `: ${slot.error}` : ''}`, automatic: true, href: slot.operation_id ? `/provisioning/${encodeURIComponent(slot.operation_id)}` : null, label: 'Previous operation' }; } const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number }; return n - (ownHandoff && slot?.healthy_at === null ? 1 : 0) >= fleetSettings().restartConcurrency ? { code: 'restart-capacity', message: `Queued to ${action}; waiting for another server to recover`, automatic: true } : null; } export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } { const slot = slotFor(server); if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 }; return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' }; }