From 8e04a190b349a60191614af62841c862d63b9ec6 Mon Sep 17 00:00:00 2001 From: josh Date: Thu, 17 Sep 2026 11:25:06 -0400 Subject: [PATCH] Roll deployment restarts through a shared fleet-wide recovery limit --- .env.example | 4 + README.md | 25 +- docker-compose.yml | 1 + komodo/stack.toml | 1 + src/lib/server/config.ts | 7 + src/lib/server/db/schema.sql | 12 + src/lib/server/pipeline/beta-runner.ts | 48 +-- src/lib/server/pipeline/restart-rollout.ts | 125 ++++++++ src/lib/server/pipeline/sweep.ts | 62 ++-- src/lib/server/pipeline/worker.ts | 10 +- src/routes/deployments/+page.svelte | 2 +- tests/unit/restart-rollout.test.ts | 333 +++++++++++++++++++++ 12 files changed, 579 insertions(+), 51 deletions(-) create mode 100644 src/lib/server/pipeline/restart-rollout.ts create mode 100644 tests/unit/restart-rollout.test.ts diff --git a/.env.example b/.env.example index ed67dfd..357e179 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,10 @@ DB_PATH=/data/cherry.db # Retention (days) for deployment rows and blob diffs DEPLOY_KEEP_DAYS=7 +# Fleet-wide rolling restart limit, shared by deployments and betas. +# A slot stays occupied until Komodo and RCON confirm recovery. +DEPLOYMENT_RESTART_CONCURRENCY=1 + # Komodo KOMODO_ADDRESS=https://komodo.example.internal KOMODO_API_KEY= diff --git a/README.md b/README.md index 59ddf85..be43ba9 100644 --- a/README.md +++ b/README.md @@ -31,13 +31,26 @@ audit diffs. ## How restarts stay safe -Every push (or beta open) creates a `deployment` row and marks each affected -server `pending`. A background sweep runs every 15s and: +File changes are staged sequentially. Their activation uses rolling restarts: +by default only **one server across the fleet** can restart or recover at a +time. Promotions, rollbacks, drift reconciliation, and beta restarts share this +limit. Set `DEPLOYMENT_RESTART_CONCURRENCY` to a positive integer to raise it. -1. Asks each server over RCON: how many players are on right now? -2. If 0 → tell Komodo to restart the server → row goes `done`. -3. If >0 → hold the row `pending` with the player count; try again next tick. -4. If RCON unreachable → hold `pending` with `error='rcon unreachable'`. +A background sweep runs every 15s. It checks recovering servers first, then +admits queued deployments in oldest-first order when there is capacity. Busy +or unreachable servers stay queued without taking a slot, so other empty +servers can proceed. A successful Komodo restart request retains its slot +until the container is healthy and RCON answers. Health checks start no sooner +than 15 seconds after the request returns, avoiding an immediate check of the +old process. An unhealthy server keeps its slot rather than letting failures +spread across the fleet. A timed-out restart also keeps its slot because Komodo +may have accepted the request. + +Slots persist in SQLite across app restarts, and overlapping timer/API triggers +share the active run. Beta options to skip health or match gates still work; +skipping a beta gate does not release the fleet's recovery slot early. New-server +creation is separate from deployment restarts and still starts its container +directly. No token, no session, no auth freeze. If Komodo or RCON aren't configured, the sweep marks pending rows `manual` so nothing hangs forever. diff --git a/docker-compose.yml b/docker-compose.yml index 80a365b..2c78a4a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,7 @@ services: BETA_ARCHIVE_DIR: /root/zombies/beta-archive DB_PATH: /data/cherry.db DEPLOY_KEEP_DAYS: "7" + DEPLOYMENT_RESTART_CONCURRENCY: ${DEPLOYMENT_RESTART_CONCURRENCY:-1} KOMODO_ADDRESS: ${KOMODO_ADDRESS} KOMODO_API_KEY: ${KOMODO_API_KEY} KOMODO_API_SECRET: ${KOMODO_API_SECRET} diff --git a/komodo/stack.toml b/komodo/stack.toml index 71b8cef..0249067 100644 --- a/komodo/stack.toml +++ b/komodo/stack.toml @@ -26,6 +26,7 @@ REL_DEST = "runtime/plutonium/storage/t5/raw/scripts/sp" BETA_ARCHIVE_DIR = "/root/zombies/beta-archive" DB_PATH = "/data/cherry.db" DEPLOY_KEEP_DAYS = "7" +DEPLOYMENT_RESTART_CONCURRENCY = "1" # secrets: bind via Komodo's env-file or secret store KOMODO_ADDRESS = "https://komodo.example.internal" diff --git a/src/lib/server/config.ts b/src/lib/server/config.ts index fb60eb0..f8587da 100644 --- a/src/lib/server/config.ts +++ b/src/lib/server/config.ts @@ -14,6 +14,12 @@ function num(name: string, fallback: number): number { return n; } +function positiveInt(name: string, fallback: number): number { + const value = num(name, fallback); + if (!Number.isSafeInteger(value) || value < 1) throw new Error(`env ${name} must be a positive integer`); + return value; +} + export const config = { serversRoot: req('SERVERS_ROOT', '/root/zombies/t5'), relDest: req('REL_DEST', 'runtime/plutonium/storage/t5/raw/scripts/sp'), @@ -29,6 +35,7 @@ export const config = { betaArchiveDir: req('BETA_ARCHIVE_DIR', '/root/zombies/beta-archive'), dbPath: req('DB_PATH', './data/cherry.db'), deployKeepDays: num('DEPLOY_KEEP_DAYS', 7), + restartConcurrency: positiveInt('DEPLOYMENT_RESTART_CONCURRENCY', 1), httpTimeoutMs: num('HTTP_TIMEOUT_MS', 10_000), komodo: { address: env.KOMODO_ADDRESS ?? '', diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index 6e0a6f1..9410983 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -199,6 +199,18 @@ CREATE TABLE IF NOT EXISTS deployment_servers ( CREATE INDEX IF NOT EXISTS idx_deployment_servers_state ON deployment_servers(restart_state); +-- Fleet-wide restart slots shared by deployments and betas. A successful API +-- response does not release a slot: the server must recover first. Persisting +-- these rows also protects the fleet across Electric Cherry process restarts. +CREATE TABLE IF NOT EXISTS restart_rollout ( + server TEXT PRIMARY KEY REFERENCES servers(name) ON DELETE CASCADE, + requested_at TEXT NOT NULL, + checked_at TEXT, + healthy_at TEXT, + players INTEGER, + error TEXT +); + CREATE TABLE IF NOT EXISTS deployment_files ( id INTEGER PRIMARY KEY AUTOINCREMENT, deployment_id TEXT NOT NULL REFERENCES deployments(id) ON DELETE CASCADE, diff --git a/src/lib/server/pipeline/beta-runner.ts b/src/lib/server/pipeline/beta-runner.ts index c54ff09..087e79a 100644 --- a/src/lib/server/pipeline/beta-runner.ts +++ b/src/lib/server/pipeline/beta-runner.ts @@ -2,9 +2,9 @@ import { db } from '../db/index.js'; import { emit } from './bus.js'; import type { BetaStage } from './bus.js'; import { komodoConfigured, rconConfigured } from '../config.js'; -import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js'; -import { komodoRestart, komodoInspectHealthy } from '../integrations/komodo.js'; +import { rconGetPlayerCount } from '../integrations/rcon.js'; import { scanForMatchEnd } from '../fleet/match-log.js'; +import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js'; interface BetaServerActiveRow { beta_id: number; @@ -84,8 +84,17 @@ function reconcileBetaState(betaId: number): void { } } -export async function betaTick(): Promise<{ processed: number }> { +let betaInFlight: Promise<{ processed: number }> | null = null; + +export function betaTick(): Promise<{ processed: number }> { + if (betaInFlight) return betaInFlight; + betaInFlight = runBetaTick().finally(() => { betaInFlight = null; }); + return betaInFlight; +} + +async function runBetaTick(): Promise<{ processed: number }> { if (!integrationsOn()) return { processed: 0 }; + await refreshRestartRollout(); const rows = db() .prepare( @@ -95,7 +104,8 @@ export async function betaTick(): Promise<{ processed: number }> { FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE b.state = 'running' - AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match')` + AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match') + ORDER BY b.id, bs.server` ) .all() as BetaServerActiveRow[]; @@ -117,19 +127,24 @@ export async function betaTick(): Promise<{ processed: number }> { async function advance(row: BetaServerActiveRow): Promise { switch (row.stage) { - case 'awaiting-empty': { + case 'awaiting-empty': + case 'restarting': { + const waitReason = rolloutWaitReason(row.server); + if (waitReason) { + transition(row.beta_id, row.server, 'awaiting-empty', waitReason); + return; + } const players = await rconGetPlayerCount(row.server); if (players === null) return; // hold — RCON unreachable, try next tick if (players > 0) return; // hold — still populated transition(row.beta_id, row.server, 'restarting'); - // Fall through to restart on the same tick for lower latency. - await advance({ ...row, stage: 'restarting' }); - return; - } - case 'restarting': { - const r = await komodoRestart(row.server); - if (!r.ok) { - transition(row.beta_id, row.server, 'failed', `komodo: ${r.message}`); + const r = await restartInRollout(row.server); + if (r.kind === 'queued') { + transition(row.beta_id, row.server, 'awaiting-empty', r.message); + return; + } + if (r.kind === 'failed') { + transition(row.beta_id, row.server, 'failed', r.message); return; } const next = row.wait_for_health ? 'awaiting-health' : row.wait_for_match ? 'awaiting-match' : 'ready'; @@ -137,14 +152,13 @@ async function advance(row: BetaServerActiveRow): Promise { return; } case 'awaiting-health': { - const h = await komodoInspectHealthy(row.server); + const h = rolloutHealth(row.server); if (!h.ok) { // Record the failure reason (including any snippet) so the // beta detail UI can surface why we're stuck. - const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : ''; setStage().run( 'awaiting-health', - `unhealthy: ${h.reason}${snippet}`, + h.reason, nowIso(), 'awaiting-health', 'awaiting-health', @@ -159,8 +173,6 @@ async function advance(row: BetaServerActiveRow): Promise { ); return; } - const rcon = await rconStatusResult(row.server); - if (!rcon.ok) return; // container reports up but game not answering yet const next = row.wait_for_match ? 'awaiting-match' : 'ready'; transition(row.beta_id, row.server, next); return; diff --git a/src/lib/server/pipeline/restart-rollout.ts b/src/lib/server/pipeline/restart-rollout.ts new file mode 100644 index 0000000..b2e2904 --- /dev/null +++ b/src/lib/server/pipeline/restart-rollout.ts @@ -0,0 +1,125 @@ +import { config } from '../config.js'; +import { db } from '../db/index.js'; +import { komodoInspectHealthy, komodoRestart } from '../integrations/komodo.js'; +import { rconStatusResult } from '../integrations/rcon.js'; + +// Komodo queues restart work. Don't mistake the still-running old container +// for a completed restart when an API kick immediately follows the request. +const HEALTH_POLL_MS = 15_000; +const requestsInFlight = new Set(); + +interface RestartSlot { + server: string; + requested_at: string; + checked_at: string | null; + healthy_at: string | null; + players: number | null; + error: string | null; +} + +function slotFor(server: string): RestartSlot | undefined { + return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined; +} + +export function rolloutWaitReason(server: string): string | null { + const slot = slotFor(server); + if (slot?.healthy_at === null) { + return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`; + } + const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number }; + return n >= config.restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null; +} + +export type RolloutRestartResult = + | { kind: 'started' } + | { kind: 'queued'; message: string } + | { kind: 'failed'; message: string }; + +/** Reserve capacity before any async I/O; callers still check for players first. */ +export async function restartInRollout(server: string): Promise { + const waitReason = db().transaction(() => { + const reason = rolloutWaitReason(server); + if (reason) return reason; + db().prepare(`INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?) + ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at, + checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`) + .run(server, new Date().toISOString()); + return null; + }).immediate(); + if (waitReason) return { kind: 'queued', message: waitReason }; + + requestsInFlight.add(server); + try { + const result = await komodoRestart(server); + if (result.ok) return { kind: 'started' }; + // These failures cannot have queued a remote restart. Other failures + // (especially a timeout) may have: retain capacity until recovery. + if (result.kind === 'no-name' || result.kind === 'unconfigured') { + db().prepare('DELETE FROM restart_rollout WHERE server = ?').run(server); + } else { + db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server); + } + return { kind: 'failed', message: `komodo: ${result.message}` }; + } catch (err) { + const message = `komodo: ${(err as Error).message}`; + db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server); + return { kind: 'failed', message }; + } finally { + // Start the grace period at completion, including slow/failed requests. + db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ?') + .run(new Date().toISOString(), server); + requestsInFlight.delete(server); + } +} + +let refreshInFlight: Promise | null = null; + +/** Poll all occupied slots, even if their originating beta was already closed. */ +export function refreshRestartRollout(): Promise { + if (refreshInFlight) return refreshInFlight; + refreshInFlight = refresh().finally(() => { refreshInFlight = null; }); + return refreshInFlight; +} + +async function refresh(): Promise { + // Adopt work started before this feature was installed. INSERT OR IGNORE + // preserves completed slots and never resets a newer restart's progress. + db().prepare(`INSERT OR IGNORE INTO restart_rollout (server, requested_at) + SELECT server, MAX(restarted_at) FROM deployment_servers + WHERE restart_state IN ('pending','failed') AND restarted_at IS NOT NULL GROUP BY server`).run(); + db().prepare(`INSERT OR IGNORE INTO restart_rollout (server, requested_at) + SELECT bs.server, MAX(COALESCE(bs.restarted_at, bs.updated_at)) + FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id + WHERE b.state = 'running' AND bs.stage IN ('restarting','awaiting-health') GROUP BY bs.server`).run(); + + const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server') + .all() as RestartSlot[]; + for (const slot of slots) { + if (requestsInFlight.has(slot.server)) continue; + if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue; + let error: string | null = null; + let players: number | null = null; + try { + const health = await komodoInspectHealthy(slot.server); + if (!health.ok) { + error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`; + } else { + const rcon = await rconStatusResult(slot.server); + if (!rcon.ok) error = `rcon not answering: ${rcon.reason}`; + else players = rcon.status.players; + } + } catch (err) { + error = `health check: ${(err as Error).message}`; + } + const now = new Date().toISOString(); + db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ? + WHERE server = ? AND requested_at = ? AND healthy_at IS NULL`) + .run(now, error ? null : now, players, error, slot.server, slot.requested_at); + } +} + +export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } { + const slot = slotFor(server); + if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 }; + return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' }; +} diff --git a/src/lib/server/pipeline/sweep.ts b/src/lib/server/pipeline/sweep.ts index 53a2190..5e66c33 100644 --- a/src/lib/server/pipeline/sweep.ts +++ b/src/lib/server/pipeline/sweep.ts @@ -1,17 +1,21 @@ import { config, komodoConfigured, rconConfigured } from '../config.js'; import { db } from '../db/index.js'; import { emit } from './bus.js'; -import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js'; -import { komodoRestart, komodoInspectHealthy } from '../integrations/komodo.js'; +import { rconGetPlayerCount } from '../integrations/rcon.js'; import { setServerField } from '../fleet/servers.js'; +import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js'; interface PendingRow { deployment_id: string; server: string; attempts: number; restarted_at: string | null; + files_changed: number; } +type SweepResult = { checked: number; restarted: number; healthy: number; held: number; failed: number }; +let sweepInFlight: Promise | null = null; + function nowIso(): string { return new Date().toISOString(); } @@ -35,7 +39,13 @@ function integrationsOn(): boolean { * Same-server dedup is per-tick per-phase so multiple concurrent deployments * targeting the same server only cost one RCON+Komodo round trip per tick. */ -export async function sweep(): Promise<{ checked: number; restarted: number; healthy: number; held: number; failed: number }> { +export function sweep(): Promise { + if (sweepInFlight) return sweepInFlight; + sweepInFlight = runSweep().finally(() => { sweepInFlight = null; }); + return sweepInFlight; +} + +async function runSweep(): Promise { if (!integrationsOn()) { // Freeze pending rows as manual — they won't drain without integrations. db() @@ -49,11 +59,15 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 }; } + // Recovery frees capacity before admitting any more servers. This runs + // even without pending deployment rows (betas share the same slots). + await refreshRestartRollout(); const pending = db() .prepare( - `SELECT deployment_id, server, attempts, restarted_at - FROM deployment_servers - WHERE restart_state IN ('pending','failed')` + `SELECT ds.deployment_id, ds.server, ds.attempts, ds.restarted_at, ds.files_changed + FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id + WHERE ds.restart_state IN ('pending','failed') + ORDER BY d.started_at, d.id, ds.server` ) .all() as PendingRow[]; if (pending.length === 0) { @@ -65,6 +79,7 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea // Split dedup caches by phase — the two phases talk to different endpoints. const preRestart = new Map(); const postRestart = new Map(); + const recoveringServers = new Set(pending.filter((row) => row.restarted_at).map((row) => row.server)); let checked = 0; let restarted = 0; @@ -80,13 +95,18 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea ); for (const row of pending) { + // A failed file write isn't a restart request. Preserve the original + // failure until the operator repairs/retries that deployment. + if (row.files_changed === 0) continue; const cache = row.restarted_at ? postRestart : preRestart; let outcome = cache.get(row.server); if (!outcome) { checked++; outcome = row.restarted_at - ? await runHealthPhase(row.server) - : await runRestartPhase(row.server); + ? runHealthPhase(row.server) + : recoveringServers.has(row.server) + ? { state: 'pending', players: null, error: 'Queued behind an earlier restart of this server', restartedNow: false } + : await runRestartPhase(row.server); cache.set(row.server, outcome); if (outcome.state === 'done' && row.restarted_at) healthy++; if (outcome.restartedNow) restarted++; @@ -152,6 +172,8 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea // Phase 1: wait for the server to be empty, then Komodo-restart it. async function runRestartPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> { + const waitReason = rolloutWaitReason(server); + if (waitReason) return { state: 'pending', players: null, error: waitReason, restartedNow: false }; const players = await rconGetPlayerCount(server); if (players === null) { return { state: 'pending', players: null, error: 'rcon unreachable', restartedNow: false }; @@ -159,32 +181,22 @@ async function runRestartPhase(server: string): Promise<{ state: 'done' | 'faile if (players > 0) { return { state: 'pending', players, error: null, restartedNow: false }; } - const r = await komodoRestart(server); - if (r.ok) { + const r = await restartInRollout(server); + if (r.kind === 'started') { // Restart triggered — flip restarted_at so the next tick runs the // health phase. Still 'pending' overall until health confirms. return { state: 'pending', players: 0, error: null, restartedNow: true }; } - if (r.kind === 'no-name') { - return { state: 'failed', players: 0, error: r.message, restartedNow: false }; - } - return { state: 'failed', players: 0, error: `komodo: ${r.message}`, restartedNow: false }; + return { state: r.kind === 'queued' ? 'pending' : 'failed', players: 0, error: r.message, restartedNow: false }; } // Phase 2: after Komodo-restart, wait until the container reports running AND // RCON is answering. Only then does the deployment_server row transition to // 'done'. If either check fails we stay pending and try again next tick. -async function runHealthPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> { - const h = await komodoInspectHealthy(server); +function runHealthPhase(server: string): { state: 'done' | 'pending'; players: number | null; error: string | null; restartedNow: boolean } { + const h = rolloutHealth(server); if (!h.ok) { - // Include a truncated response snippet (when the extractor is the - // failure) so the operator can spot which Komodo field to teach. - const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : ''; - return { state: 'pending', players: null, error: `unhealthy: ${h.reason}${snippet}`, restartedNow: false }; + return { state: 'pending', players: null, error: h.reason, restartedNow: false }; } - const rcon = await rconStatusResult(server); - if (!rcon.ok) { - return { state: 'pending', players: null, error: `rcon not answering: ${rcon.reason}`, restartedNow: false }; - } - return { state: 'done', players: rcon.status.players, error: null, restartedNow: false }; + return { state: 'done', players: h.players, error: null, restartedNow: false }; } diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index b3ff594..42c0520 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -18,7 +18,15 @@ let started = false; let tickTimer: NodeJS.Timeout | null = null; let statusTimer: NodeJS.Timeout | null = null; -async function tick(): Promise { +let tickInFlight: Promise | null = null; + +function tick(): Promise { + if (tickInFlight) return tickInFlight; + tickInFlight = runTick().finally(() => { tickInFlight = null; }); + return tickInFlight; +} + +async function runTick(): Promise { try { await sweep(); } catch (err) { diff --git a/src/routes/deployments/+page.svelte b/src/routes/deployments/+page.svelte index 7337fed..9fddc82 100644 --- a/src/routes/deployments/+page.svelte +++ b/src/routes/deployments/+page.svelte @@ -204,7 +204,7 @@
{#if loading} diff --git a/tests/unit/restart-rollout.test.ts b/tests/unit/restart-rollout.test.ts new file mode 100644 index 0000000..59317bb --- /dev/null +++ b/tests/unit/restart-rollout.test.ts @@ -0,0 +1,333 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ + komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() +})); +vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ + rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() +})); + +let fixture: Fixture; +const servers = ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_theater_3']; +const healthyRcon = { + ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '', + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } +}; + +beforeEach(async () => { + vi.resetModules(); + vi.resetAllMocks(); + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(new Date('2026-09-17T15:00:00Z')); + fixture = await newFixture(servers); + vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); + vi.stubEnv('KOMODO_API_KEY', 'test'); + vi.stubEnv('KOMODO_API_SECRET', 'test'); + vi.stubEnv('RCON_PASSWORD', 'test'); + vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1'); + const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); + await seedServerRegistry(); + const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(komodoRestart).mockResolvedValue({ ok: true }); + vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + vi.mocked(rconGetPlayerCount).mockResolvedValue(0); + vi.mocked(rconStatusResult).mockResolvedValue(healthyRcon); +}); + +afterEach(async () => { + await fixture.cleanup(); + vi.unstubAllEnvs(); + vi.useRealTimers(); +}); + +function nextTick() { vi.setSystemTime(Date.now() + 15_000); } + +async function deployment(id = 'dep-1', targets = servers, action = 'promote') { + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare('INSERT INTO deployments (id, action, started_at) VALUES (?, ?, ?)') + .run(id, action, new Date().toISOString()); + for (const server of targets) { + db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) + VALUES (?, ?, 1, 'pending')`).run(id, server); + } +} + +async function rows(id = 'dep-1') { + const { db } = await import('../../src/lib/server/db/index.js'); + return db().prepare('SELECT server, restart_state, restarted_at, error FROM deployment_servers WHERE deployment_id = ? ORDER BY server') + .all(id) as Array<{ server: string; restart_state: string; restarted_at: string | null; error: string | null }>; +} + +async function beta(targets: string[], waitForHealth = true) { + const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); + const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); + const { script } = createScript({ name: 'test script', targetRelPath: 'rollout.gsc', content: Buffer.from('main(){}') }); + return openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: getLatestVersion(script.id)!.id }], + servers: targets, waitForHealth, waitForMatch: false }); +} + +describe('rolling deployment restarts', () => { + it('restarts one server per recovery and finishes only after the last server is healthy', async () => { + await deployment(); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + expect((await sweep()).restarted).toBe(1); + expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]); + expect((await rows()).map((row) => !!row.restarted_at)).toEqual([true, false, false]); + await sweep(); // Repeated kick can't bypass the grace period. + expect(komodoRestart).toHaveBeenCalledTimes(1); + nextTick(); + expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 }); + expect((await rows()).map((row) => row.restart_state)).toEqual(['done', 'pending', 'pending']); + nextTick(); + expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 }); + nextTick(); + expect(await sweep()).toMatchObject({ restarted: 0, healthy: 1 }); + expect((await rows()).every((row) => row.restart_state === 'done')).toBe(true); + expect(vi.mocked(komodoRestart).mock.calls).toEqual(servers.map((server) => [server])); + expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get('dep-1')) + .toMatchObject({ finished_at: expect.any(String) }); + }); + + it('shares capacity across promote, rollback, and drift deployments in oldest-first order', async () => { + await deployment('z-oldest', [servers[2]], 'rollback'); + nextTick(); + await deployment('a-next', [servers[0]], 'drift'); + nextTick(); + await deployment('b-latest', [servers[1]], 'promote'); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + await sweep(); + expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]); + nextTick(); + await sweep(); + expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]], [servers[0]]]); + }); + + it('skips occupied and unreachable servers without consuming capacity', async () => { + await deployment(); + const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(4).mockResolvedValueOnce(null).mockResolvedValueOnce(0); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + await sweep(); + expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]); + }); + + it('keeps the fleet queued while either container or RCON health is failing', async () => { + await deployment(); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); + await sweep(); + nextTick(); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }); + await sweep(); + expect(komodoRestart).toHaveBeenCalledTimes(1); + nextTick(); + vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'starting' }); + await sweep(); + expect(komodoRestart).toHaveBeenCalledTimes(1); + nextTick(); + await sweep(); + expect(komodoRestart).toHaveBeenCalledTimes(2); + }); + + it('coalesces overlapping sweeps while the remote restart is unresolved', async () => { + await deployment(); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + let finish!: () => void; + const entered = new Promise((resolve) => { + vi.mocked(komodoRestart).mockImplementationOnce(() => { + resolve(); + return new Promise((done) => { finish = () => done({ ok: true }); }); + }); + }); + const first = sweep(); + await entered; + const second = sweep(); + expect(second).toBe(first); + // Even a long-running restart request cannot be freed by a health poll. + nextTick(); + const { refreshRestartRollout, rolloutWaitReason } = await import('../../src/lib/server/pipeline/restart-rollout.js'); + await refreshRestartRollout(); + expect(rolloutWaitReason(servers[1])).toMatch(/Queued/); + finish(); + await Promise.all([first, second]); + expect(komodoRestart).toHaveBeenCalledTimes(1); + }); + + it('deduplicates simultaneous deployment rows for the same server', async () => { + await deployment('dep-1', [servers[0]]); + await deployment('dep-2', [servers[0], servers[1]]); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + await sweep(); + expect(komodoRestart).toHaveBeenCalledTimes(1); + expect((await rows('dep-1'))[0].restarted_at).toBeTruthy(); + expect((await rows('dep-2'))[0].restarted_at).toBeTruthy(); + }); + + it('does not count a later deployment as having been applied by an earlier restart', async () => { + await deployment('dep-1', [servers[0]]); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + await sweep(); + await deployment('dep-2', [servers[0]]); + nextTick(); + await sweep(); + expect((await rows('dep-1'))[0].restart_state).toBe('done'); + expect((await rows('dep-2'))[0].restarted_at).toBeNull(); + await sweep(); + expect(komodoRestart).toHaveBeenCalledTimes(2); + expect((await rows('dep-2'))[0].restarted_at).toBeTruthy(); + }); + + it('preserves occupied slots across app/module restarts', async () => { + await deployment(); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + await sweep(); + const { closeDb } = await import('../../src/lib/server/db/index.js'); + closeDb(); + vi.resetModules(); + const fresh = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + await fresh.sweep(); + expect(komodoRestart).toHaveBeenCalledTimes(1); + nextTick(); + await fresh.sweep(); + expect(komodoRestart).toHaveBeenCalledTimes(2); + }); + + it('adopts pre-upgrade restarts instead of starting a second server', async () => { + await deployment(); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare('UPDATE deployment_servers SET restarted_at = ? WHERE server = ?') + .run(new Date().toISOString(), servers[0]); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + await sweep(); + expect(komodoRestart).not.toHaveBeenCalled(); + }); + + it('retains capacity after a timeout because the remote restart may have been accepted', async () => { + await deployment(); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timed out' }); + await sweep(); + expect((await rows())[0].restart_state).toBe('failed'); + expect(komodoRestart).toHaveBeenCalledTimes(1); + nextTick(); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'restarting' }); + await sweep(); + expect(komodoRestart).toHaveBeenCalledTimes(1); + }); + + it('does not restart a server whose file deployment failed', async () => { + await deployment(); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("UPDATE deployment_servers SET files_changed = 0, restart_state = 'failed', error = 'write failed' WHERE server = ?") + .run(servers[0]); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + await sweep(); + expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[1]]]); + expect((await rows())[0].error).toBe('write failed'); + }); + + it('releases slots for definite local failures that could not have started a restart', async () => { + await deployment(); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'no-name', message: 'no komodo_name' }); + await sweep(); + expect((await rows())[0].restart_state).toBe('failed'); + expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]], [servers[1]]]); + }); +}); + +describe('beta and deployment coordination', () => { + it('limits beta restarts and releases capacity only after recovery', async () => { + await beta(servers); + const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + const first = betaTick(); + expect(betaTick()).toBe(first); + await first; + expect(komodoRestart).toHaveBeenCalledTimes(1); + nextTick(); + await betaTick(); + expect(komodoRestart).toHaveBeenCalledTimes(2); + }); + + it('shares the same capacity when beta and deployment ticks run concurrently', async () => { + await beta([servers[1]]); + await deployment('dep-1', [servers[0]]); + const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js'); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + await Promise.all([sweep(), betaTick()]); + expect(komodoRestart).toHaveBeenCalledTimes(1); + }); + + it('retains beta recovery slots even when the beta skips health checks or is closed', async () => { + const { betaId } = await beta([servers[0]], false); + const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js'); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + await betaTick(); + expect(db().prepare('SELECT stage FROM beta_servers WHERE beta_id = ?').get(betaId)).toEqual({ stage: 'ready' }); + db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(betaId); + await deployment('dep-1', [servers[1]]); + await sweep(); + expect(komodoRestart).toHaveBeenCalledTimes(1); + nextTick(); + await sweep(); + expect(komodoRestart).toHaveBeenCalledTimes(2); + }); + + it('checks player occupancy again when resuming a legacy restarting beta', async () => { + const { betaId } = await beta([servers[0]]); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("UPDATE beta_servers SET stage = 'restarting', updated_at = ? WHERE beta_id = ?") + .run(new Date(Date.now() - 30_000).toISOString(), betaId); + const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rconGetPlayerCount).mockResolvedValue(3); + await betaTick(); + expect(komodoRestart).not.toHaveBeenCalled(); + }); +}); + +describe('rollout configuration', () => { + it('honors a configured limit larger than one', async () => { + const { closeDb } = await import('../../src/lib/server/db/index.js'); + closeDb(); + vi.resetModules(); + vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2'); + await deployment(); + const { sweep } = await import('../../src/lib/server/pipeline/sweep.js'); + const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + await sweep(); + expect(komodoRestart).toHaveBeenCalledTimes(2); + await sweep(); + expect(komodoRestart).toHaveBeenCalledTimes(2); + }); + + it.each(['0', '-1', '1.5', 'abc'])('rejects invalid concurrency %s', async (value) => { + const { closeDb } = await import('../../src/lib/server/db/index.js'); + closeDb(); + vi.resetModules(); + vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', value); + await expect(import('../../src/lib/server/config.js')).rejects.toThrow(/DEPLOYMENT_RESTART_CONCURRENCY/); + vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1'); + vi.resetModules(); + }); +});