Roll deployment restarts through a shared fleet-wide recovery limit

src/lib/server/pipeline/restart-rollout.ts

@@ -0,0 +1,125 @@
+import { config } from '../config.js';
+import { db } from '../db/index.js';
+import { komodoInspectHealthy, komodoRestart } from '../integrations/komodo.js';
+import { rconStatusResult } from '../integrations/rcon.js';
+
+// Komodo queues restart work. Don't mistake the still-running old container
+// for a completed restart when an API kick immediately follows the request.
+const HEALTH_POLL_MS = 15_000;
+const requestsInFlight = new Set<string>();
+
+interface RestartSlot {
+    server: string;
+    requested_at: string;
+    checked_at: string | null;
+    healthy_at: string | null;
+    players: number | null;
+    error: string | null;
+}
+
+function slotFor(server: string): RestartSlot | undefined {
+    return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
+}
+
+export function rolloutWaitReason(server: string): string | null {
+    const slot = slotFor(server);
+    if (slot?.healthy_at === null) {
+        return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
+    }
+    const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
+    return n >= config.restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null;
+}
+
+export type RolloutRestartResult =
+    | { kind: 'started' }
+    | { kind: 'queued'; message: string }
+    | { kind: 'failed'; message: string };
+
+/** Reserve capacity before any async I/O; callers still check for players first. */
+export async function restartInRollout(server: string): Promise<RolloutRestartResult> {
+    const waitReason = db().transaction(() => {
+        const reason = rolloutWaitReason(server);
+        if (reason) return reason;
+        db().prepare(`INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)
+            ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
+                checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`)
+            .run(server, new Date().toISOString());
+        return null;
+    }).immediate();
+    if (waitReason) return { kind: 'queued', message: waitReason };
+
+    requestsInFlight.add(server);
+    try {
+        const result = await komodoRestart(server);
+        if (result.ok) return { kind: 'started' };
+        // These failures cannot have queued a remote restart. Other failures
+        // (especially a timeout) may have: retain capacity until recovery.
+        if (result.kind === 'no-name' || result.kind === 'unconfigured') {
+            db().prepare('DELETE FROM restart_rollout WHERE server = ?').run(server);
+        } else {
+            db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server);
+        }
+        return { kind: 'failed', message: `komodo: ${result.message}` };
+    } catch (err) {
+        const message = `komodo: ${(err as Error).message}`;
+        db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server);
+        return { kind: 'failed', message };
+    } finally {
+        // Start the grace period at completion, including slow/failed requests.
+        db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ?')
+            .run(new Date().toISOString(), server);
+        requestsInFlight.delete(server);
+    }
+}
+
+let refreshInFlight: Promise<void> | null = null;
+
+/** Poll all occupied slots, even if their originating beta was already closed. */
+export function refreshRestartRollout(): Promise<void> {
+    if (refreshInFlight) return refreshInFlight;
+    refreshInFlight = refresh().finally(() => { refreshInFlight = null; });
+    return refreshInFlight;
+}
+
+async function refresh(): Promise<void> {
+    // Adopt work started before this feature was installed. INSERT OR IGNORE
+    // preserves completed slots and never resets a newer restart's progress.
+    db().prepare(`INSERT OR IGNORE INTO restart_rollout (server, requested_at)
+        SELECT server, MAX(restarted_at) FROM deployment_servers
+        WHERE restart_state IN ('pending','failed') AND restarted_at IS NOT NULL GROUP BY server`).run();
+    db().prepare(`INSERT OR IGNORE INTO restart_rollout (server, requested_at)
+        SELECT bs.server, MAX(COALESCE(bs.restarted_at, bs.updated_at))
+        FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
+        WHERE b.state = 'running' AND bs.stage IN ('restarting','awaiting-health') GROUP BY bs.server`).run();
+
+    const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server')
+        .all() as RestartSlot[];
+    for (const slot of slots) {
+        if (requestsInFlight.has(slot.server)) continue;
+        if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
+        let error: string | null = null;
+        let players: number | null = null;
+        try {
+            const health = await komodoInspectHealthy(slot.server);
+            if (!health.ok) {
+                error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
+            } else {
+                const rcon = await rconStatusResult(slot.server);
+                if (!rcon.ok) error = `rcon not answering: ${rcon.reason}`;
+                else players = rcon.status.players;
+            }
+        } catch (err) {
+            error = `health check: ${(err as Error).message}`;
+        }
+        const now = new Date().toISOString();
+        db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
+            WHERE server = ? AND requested_at = ? AND healthy_at IS NULL`)
+            .run(now, error ? null : now, players, error, slot.server, slot.requested_at);
+    }
+}
+
+export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } {
+    const slot = slotFor(server);
+    if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 };
+    return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' };
+}