Roll deployment restarts through a shared fleet-wide recovery limit
This commit is contained in:
@@ -10,6 +10,10 @@ DB_PATH=/data/cherry.db
|
||||
# Retention (days) for deployment rows and blob diffs
|
||||
DEPLOY_KEEP_DAYS=7
|
||||
|
||||
# Fleet-wide rolling restart limit, shared by deployments and betas.
|
||||
# A slot stays occupied until Komodo and RCON confirm recovery.
|
||||
DEPLOYMENT_RESTART_CONCURRENCY=1
|
||||
|
||||
# Komodo
|
||||
KOMODO_ADDRESS=https://komodo.example.internal
|
||||
KOMODO_API_KEY=
|
||||
|
||||
@@ -31,13 +31,26 @@ audit diffs.
|
||||
|
||||
## How restarts stay safe
|
||||
|
||||
Every push (or beta open) creates a `deployment` row and marks each affected
|
||||
server `pending`. A background sweep runs every 15s and:
|
||||
File changes are staged sequentially. Their activation uses rolling restarts:
|
||||
by default only **one server across the fleet** can restart or recover at a
|
||||
time. Promotions, rollbacks, drift reconciliation, and beta restarts share this
|
||||
limit. Set `DEPLOYMENT_RESTART_CONCURRENCY` to a positive integer to raise it.
|
||||
|
||||
1. Asks each server over RCON: how many players are on right now?
|
||||
2. If 0 → tell Komodo to restart the server → row goes `done`.
|
||||
3. If >0 → hold the row `pending` with the player count; try again next tick.
|
||||
4. If RCON unreachable → hold `pending` with `error='rcon unreachable'`.
|
||||
A background sweep runs every 15s. It checks recovering servers first, then
|
||||
admits queued deployments in oldest-first order when there is capacity. Busy
|
||||
or unreachable servers stay queued without taking a slot, so other empty
|
||||
servers can proceed. A successful Komodo restart request retains its slot
|
||||
until the container is healthy and RCON answers. Health checks start no sooner
|
||||
than 15 seconds after the request returns, avoiding an immediate check of the
|
||||
old process. An unhealthy server keeps its slot rather than letting failures
|
||||
spread across the fleet. A timed-out restart also keeps its slot because Komodo
|
||||
may have accepted the request.
|
||||
|
||||
Slots persist in SQLite across app restarts, and overlapping timer/API triggers
|
||||
share the active run. Beta options to skip health or match gates still work;
|
||||
skipping a beta gate does not release the fleet's recovery slot early. New-server
|
||||
creation is separate from deployment restarts and still starts its container
|
||||
directly.
|
||||
|
||||
No token, no session, no auth freeze. If Komodo or RCON aren't configured,
|
||||
the sweep marks pending rows `manual` so nothing hangs forever.
|
||||
|
||||
@@ -31,6 +31,7 @@ services:
|
||||
BETA_ARCHIVE_DIR: /root/zombies/beta-archive
|
||||
DB_PATH: /data/cherry.db
|
||||
DEPLOY_KEEP_DAYS: "7"
|
||||
DEPLOYMENT_RESTART_CONCURRENCY: ${DEPLOYMENT_RESTART_CONCURRENCY:-1}
|
||||
KOMODO_ADDRESS: ${KOMODO_ADDRESS}
|
||||
KOMODO_API_KEY: ${KOMODO_API_KEY}
|
||||
KOMODO_API_SECRET: ${KOMODO_API_SECRET}
|
||||
|
||||
@@ -26,6 +26,7 @@ REL_DEST = "runtime/plutonium/storage/t5/raw/scripts/sp"
|
||||
BETA_ARCHIVE_DIR = "/root/zombies/beta-archive"
|
||||
DB_PATH = "/data/cherry.db"
|
||||
DEPLOY_KEEP_DAYS = "7"
|
||||
DEPLOYMENT_RESTART_CONCURRENCY = "1"
|
||||
|
||||
# secrets: bind via Komodo's env-file or secret store
|
||||
KOMODO_ADDRESS = "https://komodo.example.internal"
|
||||
|
||||
@@ -14,6 +14,12 @@ function num(name: string, fallback: number): number {
|
||||
return n;
|
||||
}
|
||||
|
||||
function positiveInt(name: string, fallback: number): number {
|
||||
const value = num(name, fallback);
|
||||
if (!Number.isSafeInteger(value) || value < 1) throw new Error(`env ${name} must be a positive integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
export const config = {
|
||||
serversRoot: req('SERVERS_ROOT', '/root/zombies/t5'),
|
||||
relDest: req('REL_DEST', 'runtime/plutonium/storage/t5/raw/scripts/sp'),
|
||||
@@ -29,6 +35,7 @@ export const config = {
|
||||
betaArchiveDir: req('BETA_ARCHIVE_DIR', '/root/zombies/beta-archive'),
|
||||
dbPath: req('DB_PATH', './data/cherry.db'),
|
||||
deployKeepDays: num('DEPLOY_KEEP_DAYS', 7),
|
||||
restartConcurrency: positiveInt('DEPLOYMENT_RESTART_CONCURRENCY', 1),
|
||||
httpTimeoutMs: num('HTTP_TIMEOUT_MS', 10_000),
|
||||
komodo: {
|
||||
address: env.KOMODO_ADDRESS ?? '',
|
||||
|
||||
@@ -199,6 +199,18 @@ CREATE TABLE IF NOT EXISTS deployment_servers (
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_deployment_servers_state ON deployment_servers(restart_state);
|
||||
|
||||
-- Fleet-wide restart slots shared by deployments and betas. A successful API
|
||||
-- response does not release a slot: the server must recover first. Persisting
|
||||
-- these rows also protects the fleet across Electric Cherry process restarts.
|
||||
CREATE TABLE IF NOT EXISTS restart_rollout (
|
||||
server TEXT PRIMARY KEY REFERENCES servers(name) ON DELETE CASCADE,
|
||||
requested_at TEXT NOT NULL,
|
||||
checked_at TEXT,
|
||||
healthy_at TEXT,
|
||||
players INTEGER,
|
||||
error TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS deployment_files (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
deployment_id TEXT NOT NULL REFERENCES deployments(id) ON DELETE CASCADE,
|
||||
|
||||
@@ -2,9 +2,9 @@ import { db } from '../db/index.js';
|
||||
import { emit } from './bus.js';
|
||||
import type { BetaStage } from './bus.js';
|
||||
import { komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
|
||||
import { komodoRestart, komodoInspectHealthy } from '../integrations/komodo.js';
|
||||
import { rconGetPlayerCount } from '../integrations/rcon.js';
|
||||
import { scanForMatchEnd } from '../fleet/match-log.js';
|
||||
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
|
||||
|
||||
interface BetaServerActiveRow {
|
||||
beta_id: number;
|
||||
@@ -84,8 +84,17 @@ function reconcileBetaState(betaId: number): void {
|
||||
}
|
||||
}
|
||||
|
||||
export async function betaTick(): Promise<{ processed: number }> {
|
||||
let betaInFlight: Promise<{ processed: number }> | null = null;
|
||||
|
||||
export function betaTick(): Promise<{ processed: number }> {
|
||||
if (betaInFlight) return betaInFlight;
|
||||
betaInFlight = runBetaTick().finally(() => { betaInFlight = null; });
|
||||
return betaInFlight;
|
||||
}
|
||||
|
||||
async function runBetaTick(): Promise<{ processed: number }> {
|
||||
if (!integrationsOn()) return { processed: 0 };
|
||||
await refreshRestartRollout();
|
||||
|
||||
const rows = db()
|
||||
.prepare(
|
||||
@@ -95,7 +104,8 @@ export async function betaTick(): Promise<{ processed: number }> {
|
||||
FROM beta_servers bs
|
||||
JOIN betas b ON b.id = bs.beta_id
|
||||
WHERE b.state = 'running'
|
||||
AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match')`
|
||||
AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match')
|
||||
ORDER BY b.id, bs.server`
|
||||
)
|
||||
.all() as BetaServerActiveRow[];
|
||||
|
||||
@@ -117,19 +127,24 @@ export async function betaTick(): Promise<{ processed: number }> {
|
||||
|
||||
async function advance(row: BetaServerActiveRow): Promise<void> {
|
||||
switch (row.stage) {
|
||||
case 'awaiting-empty': {
|
||||
case 'awaiting-empty':
|
||||
case 'restarting': {
|
||||
const waitReason = rolloutWaitReason(row.server);
|
||||
if (waitReason) {
|
||||
transition(row.beta_id, row.server, 'awaiting-empty', waitReason);
|
||||
return;
|
||||
}
|
||||
const players = await rconGetPlayerCount(row.server);
|
||||
if (players === null) return; // hold — RCON unreachable, try next tick
|
||||
if (players > 0) return; // hold — still populated
|
||||
transition(row.beta_id, row.server, 'restarting');
|
||||
// Fall through to restart on the same tick for lower latency.
|
||||
await advance({ ...row, stage: 'restarting' });
|
||||
return;
|
||||
}
|
||||
case 'restarting': {
|
||||
const r = await komodoRestart(row.server);
|
||||
if (!r.ok) {
|
||||
transition(row.beta_id, row.server, 'failed', `komodo: ${r.message}`);
|
||||
const r = await restartInRollout(row.server);
|
||||
if (r.kind === 'queued') {
|
||||
transition(row.beta_id, row.server, 'awaiting-empty', r.message);
|
||||
return;
|
||||
}
|
||||
if (r.kind === 'failed') {
|
||||
transition(row.beta_id, row.server, 'failed', r.message);
|
||||
return;
|
||||
}
|
||||
const next = row.wait_for_health ? 'awaiting-health' : row.wait_for_match ? 'awaiting-match' : 'ready';
|
||||
@@ -137,14 +152,13 @@ async function advance(row: BetaServerActiveRow): Promise<void> {
|
||||
return;
|
||||
}
|
||||
case 'awaiting-health': {
|
||||
const h = await komodoInspectHealthy(row.server);
|
||||
const h = rolloutHealth(row.server);
|
||||
if (!h.ok) {
|
||||
// Record the failure reason (including any snippet) so the
|
||||
// beta detail UI can surface why we're stuck.
|
||||
const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : '';
|
||||
setStage().run(
|
||||
'awaiting-health',
|
||||
`unhealthy: ${h.reason}${snippet}`,
|
||||
h.reason,
|
||||
nowIso(),
|
||||
'awaiting-health',
|
||||
'awaiting-health',
|
||||
@@ -159,8 +173,6 @@ async function advance(row: BetaServerActiveRow): Promise<void> {
|
||||
);
|
||||
return;
|
||||
}
|
||||
const rcon = await rconStatusResult(row.server);
|
||||
if (!rcon.ok) return; // container reports up but game not answering yet
|
||||
const next = row.wait_for_match ? 'awaiting-match' : 'ready';
|
||||
transition(row.beta_id, row.server, next);
|
||||
return;
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
import { config } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoInspectHealthy, komodoRestart } from '../integrations/komodo.js';
|
||||
import { rconStatusResult } from '../integrations/rcon.js';
|
||||
|
||||
// Komodo queues restart work. Don't mistake the still-running old container
|
||||
// for a completed restart when an API kick immediately follows the request.
|
||||
const HEALTH_POLL_MS = 15_000;
|
||||
const requestsInFlight = new Set<string>();
|
||||
|
||||
interface RestartSlot {
|
||||
server: string;
|
||||
requested_at: string;
|
||||
checked_at: string | null;
|
||||
healthy_at: string | null;
|
||||
players: number | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
function slotFor(server: string): RestartSlot | undefined {
|
||||
return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
|
||||
}
|
||||
|
||||
export function rolloutWaitReason(server: string): string | null {
|
||||
const slot = slotFor(server);
|
||||
if (slot?.healthy_at === null) {
|
||||
return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
|
||||
}
|
||||
const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
|
||||
return n >= config.restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null;
|
||||
}
|
||||
|
||||
export type RolloutRestartResult =
|
||||
| { kind: 'started' }
|
||||
| { kind: 'queued'; message: string }
|
||||
| { kind: 'failed'; message: string };
|
||||
|
||||
/** Reserve capacity before any async I/O; callers still check for players first. */
|
||||
export async function restartInRollout(server: string): Promise<RolloutRestartResult> {
|
||||
const waitReason = db().transaction(() => {
|
||||
const reason = rolloutWaitReason(server);
|
||||
if (reason) return reason;
|
||||
db().prepare(`INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)
|
||||
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
|
||||
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`)
|
||||
.run(server, new Date().toISOString());
|
||||
return null;
|
||||
}).immediate();
|
||||
if (waitReason) return { kind: 'queued', message: waitReason };
|
||||
|
||||
requestsInFlight.add(server);
|
||||
try {
|
||||
const result = await komodoRestart(server);
|
||||
if (result.ok) return { kind: 'started' };
|
||||
// These failures cannot have queued a remote restart. Other failures
|
||||
// (especially a timeout) may have: retain capacity until recovery.
|
||||
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
|
||||
db().prepare('DELETE FROM restart_rollout WHERE server = ?').run(server);
|
||||
} else {
|
||||
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server);
|
||||
}
|
||||
return { kind: 'failed', message: `komodo: ${result.message}` };
|
||||
} catch (err) {
|
||||
const message = `komodo: ${(err as Error).message}`;
|
||||
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server);
|
||||
return { kind: 'failed', message };
|
||||
} finally {
|
||||
// Start the grace period at completion, including slow/failed requests.
|
||||
db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ?')
|
||||
.run(new Date().toISOString(), server);
|
||||
requestsInFlight.delete(server);
|
||||
}
|
||||
}
|
||||
|
||||
let refreshInFlight: Promise<void> | null = null;
|
||||
|
||||
/** Poll all occupied slots, even if their originating beta was already closed. */
|
||||
export function refreshRestartRollout(): Promise<void> {
|
||||
if (refreshInFlight) return refreshInFlight;
|
||||
refreshInFlight = refresh().finally(() => { refreshInFlight = null; });
|
||||
return refreshInFlight;
|
||||
}
|
||||
|
||||
async function refresh(): Promise<void> {
|
||||
// Adopt work started before this feature was installed. INSERT OR IGNORE
|
||||
// preserves completed slots and never resets a newer restart's progress.
|
||||
db().prepare(`INSERT OR IGNORE INTO restart_rollout (server, requested_at)
|
||||
SELECT server, MAX(restarted_at) FROM deployment_servers
|
||||
WHERE restart_state IN ('pending','failed') AND restarted_at IS NOT NULL GROUP BY server`).run();
|
||||
db().prepare(`INSERT OR IGNORE INTO restart_rollout (server, requested_at)
|
||||
SELECT bs.server, MAX(COALESCE(bs.restarted_at, bs.updated_at))
|
||||
FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
|
||||
WHERE b.state = 'running' AND bs.stage IN ('restarting','awaiting-health') GROUP BY bs.server`).run();
|
||||
|
||||
const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server')
|
||||
.all() as RestartSlot[];
|
||||
for (const slot of slots) {
|
||||
if (requestsInFlight.has(slot.server)) continue;
|
||||
if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
|
||||
let error: string | null = null;
|
||||
let players: number | null = null;
|
||||
try {
|
||||
const health = await komodoInspectHealthy(slot.server);
|
||||
if (!health.ok) {
|
||||
error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
|
||||
} else {
|
||||
const rcon = await rconStatusResult(slot.server);
|
||||
if (!rcon.ok) error = `rcon not answering: ${rcon.reason}`;
|
||||
else players = rcon.status.players;
|
||||
}
|
||||
} catch (err) {
|
||||
error = `health check: ${(err as Error).message}`;
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
|
||||
WHERE server = ? AND requested_at = ? AND healthy_at IS NULL`)
|
||||
.run(now, error ? null : now, players, error, slot.server, slot.requested_at);
|
||||
}
|
||||
}
|
||||
|
||||
export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } {
|
||||
const slot = slotFor(server);
|
||||
if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 };
|
||||
return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' };
|
||||
}
|
||||
@@ -1,17 +1,21 @@
|
||||
import { config, komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { emit } from './bus.js';
|
||||
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
|
||||
import { komodoRestart, komodoInspectHealthy } from '../integrations/komodo.js';
|
||||
import { rconGetPlayerCount } from '../integrations/rcon.js';
|
||||
import { setServerField } from '../fleet/servers.js';
|
||||
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
|
||||
|
||||
interface PendingRow {
|
||||
deployment_id: string;
|
||||
server: string;
|
||||
attempts: number;
|
||||
restarted_at: string | null;
|
||||
files_changed: number;
|
||||
}
|
||||
|
||||
type SweepResult = { checked: number; restarted: number; healthy: number; held: number; failed: number };
|
||||
let sweepInFlight: Promise<SweepResult> | null = null;
|
||||
|
||||
function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
@@ -35,7 +39,13 @@ function integrationsOn(): boolean {
|
||||
* Same-server dedup is per-tick per-phase so multiple concurrent deployments
|
||||
* targeting the same server only cost one RCON+Komodo round trip per tick.
|
||||
*/
|
||||
export async function sweep(): Promise<{ checked: number; restarted: number; healthy: number; held: number; failed: number }> {
|
||||
export function sweep(): Promise<SweepResult> {
|
||||
if (sweepInFlight) return sweepInFlight;
|
||||
sweepInFlight = runSweep().finally(() => { sweepInFlight = null; });
|
||||
return sweepInFlight;
|
||||
}
|
||||
|
||||
async function runSweep(): Promise<SweepResult> {
|
||||
if (!integrationsOn()) {
|
||||
// Freeze pending rows as manual — they won't drain without integrations.
|
||||
db()
|
||||
@@ -49,11 +59,15 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea
|
||||
return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
|
||||
}
|
||||
|
||||
// Recovery frees capacity before admitting any more servers. This runs
|
||||
// even without pending deployment rows (betas share the same slots).
|
||||
await refreshRestartRollout();
|
||||
const pending = db()
|
||||
.prepare(
|
||||
`SELECT deployment_id, server, attempts, restarted_at
|
||||
FROM deployment_servers
|
||||
WHERE restart_state IN ('pending','failed')`
|
||||
`SELECT ds.deployment_id, ds.server, ds.attempts, ds.restarted_at, ds.files_changed
|
||||
FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
|
||||
WHERE ds.restart_state IN ('pending','failed')
|
||||
ORDER BY d.started_at, d.id, ds.server`
|
||||
)
|
||||
.all() as PendingRow[];
|
||||
if (pending.length === 0) {
|
||||
@@ -65,6 +79,7 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea
|
||||
// Split dedup caches by phase — the two phases talk to different endpoints.
|
||||
const preRestart = new Map<string, Outcome>();
|
||||
const postRestart = new Map<string, Outcome>();
|
||||
const recoveringServers = new Set(pending.filter((row) => row.restarted_at).map((row) => row.server));
|
||||
|
||||
let checked = 0;
|
||||
let restarted = 0;
|
||||
@@ -80,13 +95,18 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea
|
||||
);
|
||||
|
||||
for (const row of pending) {
|
||||
// A failed file write isn't a restart request. Preserve the original
|
||||
// failure until the operator repairs/retries that deployment.
|
||||
if (row.files_changed === 0) continue;
|
||||
const cache = row.restarted_at ? postRestart : preRestart;
|
||||
let outcome = cache.get(row.server);
|
||||
if (!outcome) {
|
||||
checked++;
|
||||
outcome = row.restarted_at
|
||||
? await runHealthPhase(row.server)
|
||||
: await runRestartPhase(row.server);
|
||||
? runHealthPhase(row.server)
|
||||
: recoveringServers.has(row.server)
|
||||
? { state: 'pending', players: null, error: 'Queued behind an earlier restart of this server', restartedNow: false }
|
||||
: await runRestartPhase(row.server);
|
||||
cache.set(row.server, outcome);
|
||||
if (outcome.state === 'done' && row.restarted_at) healthy++;
|
||||
if (outcome.restartedNow) restarted++;
|
||||
@@ -152,6 +172,8 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea
|
||||
|
||||
// Phase 1: wait for the server to be empty, then Komodo-restart it.
|
||||
async function runRestartPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
|
||||
const waitReason = rolloutWaitReason(server);
|
||||
if (waitReason) return { state: 'pending', players: null, error: waitReason, restartedNow: false };
|
||||
const players = await rconGetPlayerCount(server);
|
||||
if (players === null) {
|
||||
return { state: 'pending', players: null, error: 'rcon unreachable', restartedNow: false };
|
||||
@@ -159,32 +181,22 @@ async function runRestartPhase(server: string): Promise<{ state: 'done' | 'faile
|
||||
if (players > 0) {
|
||||
return { state: 'pending', players, error: null, restartedNow: false };
|
||||
}
|
||||
const r = await komodoRestart(server);
|
||||
if (r.ok) {
|
||||
const r = await restartInRollout(server);
|
||||
if (r.kind === 'started') {
|
||||
// Restart triggered — flip restarted_at so the next tick runs the
|
||||
// health phase. Still 'pending' overall until health confirms.
|
||||
return { state: 'pending', players: 0, error: null, restartedNow: true };
|
||||
}
|
||||
if (r.kind === 'no-name') {
|
||||
return { state: 'failed', players: 0, error: r.message, restartedNow: false };
|
||||
}
|
||||
return { state: 'failed', players: 0, error: `komodo: ${r.message}`, restartedNow: false };
|
||||
return { state: r.kind === 'queued' ? 'pending' : 'failed', players: 0, error: r.message, restartedNow: false };
|
||||
}
|
||||
|
||||
// Phase 2: after Komodo-restart, wait until the container reports running AND
|
||||
// RCON is answering. Only then does the deployment_server row transition to
|
||||
// 'done'. If either check fails we stay pending and try again next tick.
|
||||
async function runHealthPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
|
||||
const h = await komodoInspectHealthy(server);
|
||||
function runHealthPhase(server: string): { state: 'done' | 'pending'; players: number | null; error: string | null; restartedNow: boolean } {
|
||||
const h = rolloutHealth(server);
|
||||
if (!h.ok) {
|
||||
// Include a truncated response snippet (when the extractor is the
|
||||
// failure) so the operator can spot which Komodo field to teach.
|
||||
const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : '';
|
||||
return { state: 'pending', players: null, error: `unhealthy: ${h.reason}${snippet}`, restartedNow: false };
|
||||
return { state: 'pending', players: null, error: h.reason, restartedNow: false };
|
||||
}
|
||||
const rcon = await rconStatusResult(server);
|
||||
if (!rcon.ok) {
|
||||
return { state: 'pending', players: null, error: `rcon not answering: ${rcon.reason}`, restartedNow: false };
|
||||
}
|
||||
return { state: 'done', players: rcon.status.players, error: null, restartedNow: false };
|
||||
return { state: 'done', players: h.players, error: null, restartedNow: false };
|
||||
}
|
||||
|
||||
@@ -18,7 +18,15 @@ let started = false;
|
||||
let tickTimer: NodeJS.Timeout | null = null;
|
||||
let statusTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
async function tick(): Promise<void> {
|
||||
let tickInFlight: Promise<void> | null = null;
|
||||
|
||||
function tick(): Promise<void> {
|
||||
if (tickInFlight) return tickInFlight;
|
||||
tickInFlight = runTick().finally(() => { tickInFlight = null; });
|
||||
return tickInFlight;
|
||||
}
|
||||
|
||||
async function runTick(): Promise<void> {
|
||||
try {
|
||||
await sweep();
|
||||
} catch (err) {
|
||||
|
||||
@@ -204,7 +204,7 @@
|
||||
<div class="page">
|
||||
<SectionHeader
|
||||
title="Deployments"
|
||||
subtitle="Fleet-wide script deploys, drift reconciliation, and rollbacks."
|
||||
subtitle="Fleet deployments, drift reconciliation, and rollbacks. Servers restart in stages as earlier servers recover."
|
||||
/>
|
||||
|
||||
{#if loading}
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
|
||||
}));
|
||||
|
||||
let fixture: Fixture;
|
||||
const servers = ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_theater_3'];
|
||||
const healthyRcon = {
|
||||
ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
vi.resetAllMocks();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-17T15:00:00Z'));
|
||||
fixture = await newFixture(servers);
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'test');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'test');
|
||||
vi.stubEnv('RCON_PASSWORD', 'test');
|
||||
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rconStatusResult).mockResolvedValue(healthyRcon);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fixture.cleanup();
|
||||
vi.unstubAllEnvs();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
function nextTick() { vi.setSystemTime(Date.now() + 15_000); }
|
||||
|
||||
async function deployment(id = 'dep-1', targets = servers, action = 'promote') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('INSERT INTO deployments (id, action, started_at) VALUES (?, ?, ?)')
|
||||
.run(id, action, new Date().toISOString());
|
||||
for (const server of targets) {
|
||||
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state)
|
||||
VALUES (?, ?, 1, 'pending')`).run(id, server);
|
||||
}
|
||||
}
|
||||
|
||||
async function rows(id = 'dep-1') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
return db().prepare('SELECT server, restart_state, restarted_at, error FROM deployment_servers WHERE deployment_id = ? ORDER BY server')
|
||||
.all(id) as Array<{ server: string; restart_state: string; restarted_at: string | null; error: string | null }>;
|
||||
}
|
||||
|
||||
async function beta(targets: string[], waitForHealth = true) {
|
||||
const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { script } = createScript({ name: 'test script', targetRelPath: 'rollout.gsc', content: Buffer.from('main(){}') });
|
||||
return openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: getLatestVersion(script.id)!.id }],
|
||||
servers: targets, waitForHealth, waitForMatch: false });
|
||||
}
|
||||
|
||||
describe('rolling deployment restarts', () => {
|
||||
it('restarts one server per recovery and finishes only after the last server is healthy', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect((await sweep()).restarted).toBe(1);
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
|
||||
expect((await rows()).map((row) => !!row.restarted_at)).toEqual([true, false, false]);
|
||||
await sweep(); // Repeated kick can't bypass the grace period.
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 });
|
||||
expect((await rows()).map((row) => row.restart_state)).toEqual(['done', 'pending', 'pending']);
|
||||
nextTick();
|
||||
expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 });
|
||||
nextTick();
|
||||
expect(await sweep()).toMatchObject({ restarted: 0, healthy: 1 });
|
||||
expect((await rows()).every((row) => row.restart_state === 'done')).toBe(true);
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual(servers.map((server) => [server]));
|
||||
expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get('dep-1'))
|
||||
.toMatchObject({ finished_at: expect.any(String) });
|
||||
});
|
||||
|
||||
it('shares capacity across promote, rollback, and drift deployments in oldest-first order', async () => {
|
||||
await deployment('z-oldest', [servers[2]], 'rollback');
|
||||
nextTick();
|
||||
await deployment('a-next', [servers[0]], 'drift');
|
||||
nextTick();
|
||||
await deployment('b-latest', [servers[1]], 'promote');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
|
||||
nextTick();
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]], [servers[0]]]);
|
||||
});
|
||||
|
||||
it('skips occupied and unreachable servers without consuming capacity', async () => {
|
||||
await deployment();
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(4).mockResolvedValueOnce(null).mockResolvedValueOnce(0);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
|
||||
});
|
||||
|
||||
it('keeps the fleet queued while either container or RCON health is failing', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await sweep();
|
||||
nextTick();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' });
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'starting' });
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('coalesces overlapping sweeps while the remote restart is unresolved', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
let finish!: () => void;
|
||||
const entered = new Promise<void>((resolve) => {
|
||||
vi.mocked(komodoRestart).mockImplementationOnce(() => {
|
||||
resolve();
|
||||
return new Promise((done) => { finish = () => done({ ok: true }); });
|
||||
});
|
||||
});
|
||||
const first = sweep();
|
||||
await entered;
|
||||
const second = sweep();
|
||||
expect(second).toBe(first);
|
||||
// Even a long-running restart request cannot be freed by a health poll.
|
||||
nextTick();
|
||||
const { refreshRestartRollout, rolloutWaitReason } = await import('../../src/lib/server/pipeline/restart-rollout.js');
|
||||
await refreshRestartRollout();
|
||||
expect(rolloutWaitReason(servers[1])).toMatch(/Queued/);
|
||||
finish();
|
||||
await Promise.all([first, second]);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('deduplicates simultaneous deployment rows for the same server', async () => {
|
||||
await deployment('dep-1', [servers[0]]);
|
||||
await deployment('dep-2', [servers[0], servers[1]]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
expect((await rows('dep-1'))[0].restarted_at).toBeTruthy();
|
||||
expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
|
||||
});
|
||||
|
||||
it('does not count a later deployment as having been applied by an earlier restart', async () => {
|
||||
await deployment('dep-1', [servers[0]]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
await deployment('dep-2', [servers[0]]);
|
||||
nextTick();
|
||||
await sweep();
|
||||
expect((await rows('dep-1'))[0].restart_state).toBe('done');
|
||||
expect((await rows('dep-2'))[0].restarted_at).toBeNull();
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
|
||||
});
|
||||
|
||||
it('preserves occupied slots across app/module restarts', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
await sweep();
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
vi.resetModules();
|
||||
const fresh = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await fresh.sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
await fresh.sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('adopts pre-upgrade restarts instead of starting a second server', async () => {
|
||||
await deployment();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('UPDATE deployment_servers SET restarted_at = ? WHERE server = ?')
|
||||
.run(new Date().toISOString(), servers[0]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('retains capacity after a timeout because the remote restart may have been accepted', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timed out' });
|
||||
await sweep();
|
||||
expect((await rows())[0].restart_state).toBe('failed');
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'restarting' });
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not restart a server whose file deployment failed', async () => {
|
||||
await deployment();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE deployment_servers SET files_changed = 0, restart_state = 'failed', error = 'write failed' WHERE server = ?")
|
||||
.run(servers[0]);
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[1]]]);
|
||||
expect((await rows())[0].error).toBe('write failed');
|
||||
});
|
||||
|
||||
it('releases slots for definite local failures that could not have started a restart', async () => {
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'no-name', message: 'no komodo_name' });
|
||||
await sweep();
|
||||
expect((await rows())[0].restart_state).toBe('failed');
|
||||
expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]], [servers[1]]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('beta and deployment coordination', () => {
|
||||
it('limits beta restarts and releases capacity only after recovery', async () => {
|
||||
await beta(servers);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const first = betaTick();
|
||||
expect(betaTick()).toBe(first);
|
||||
await first;
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
await betaTick();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('shares the same capacity when beta and deployment ticks run concurrently', async () => {
|
||||
await beta([servers[1]]);
|
||||
await deployment('dep-1', [servers[0]]);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await Promise.all([sweep(), betaTick()]);
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('retains beta recovery slots even when the beta skips health checks or is closed', async () => {
|
||||
const { betaId } = await beta([servers[0]], false);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await betaTick();
|
||||
expect(db().prepare('SELECT stage FROM beta_servers WHERE beta_id = ?').get(betaId)).toEqual({ stage: 'ready' });
|
||||
db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(betaId);
|
||||
await deployment('dep-1', [servers[1]]);
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
||||
nextTick();
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('checks player occupancy again when resuming a legacy restarting beta', async () => {
|
||||
const { betaId } = await beta([servers[0]]);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE beta_servers SET stage = 'restarting', updated_at = ? WHERE beta_id = ?")
|
||||
.run(new Date(Date.now() - 30_000).toISOString(), betaId);
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(3);
|
||||
await betaTick();
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('rollout configuration', () => {
|
||||
it('honors a configured limit larger than one', async () => {
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
vi.resetModules();
|
||||
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2');
|
||||
await deployment();
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
await sweep();
|
||||
expect(komodoRestart).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it.each(['0', '-1', '1.5', 'abc'])('rejects invalid concurrency %s', async (value) => {
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
vi.resetModules();
|
||||
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', value);
|
||||
await expect(import('../../src/lib/server/config.js')).rejects.toThrow(/DEPLOYMENT_RESTART_CONCURRENCY/);
|
||||
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
|
||||
vi.resetModules();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user