josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Roll deployment restarts through a shared fleet-wide recovery limit

Browse Source
This commit is contained in:
josh
2026-09-17 11:25:06 -04:00
parent 3f0d6dd87d
commit 8e04a190b3
12 changed files with 579 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/sweep.ts
+37 -25
View File
@@ -1,17 +1,21 @@
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { db } from '../db/index.js';
import { emit } from './bus.js';
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
import { komodoRestart, komodoInspectHealthy } from '../integrations/komodo.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { setServerField } from '../fleet/servers.js';
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
interface PendingRow {
deployment_id: string;
server: string;
attempts: number;
restarted_at: string | null;
files_changed: number;
}
type SweepResult = { checked: number; restarted: number; healthy: number; held: number; failed: number };
let sweepInFlight: Promise<SweepResult> | null = null;
function nowIso(): string {
return new Date().toISOString();
}
@@ -35,7 +39,13 @@ function integrationsOn(): boolean {
* Same-server dedup is per-tick per-phase so multiple concurrent deployments
* targeting the same server only cost one RCON+Komodo round trip per tick.
*/
export async function sweep(): Promise<{ checked: number; restarted: number; healthy: number; held: number; failed: number }> {
export function sweep(): Promise<SweepResult> {
if (sweepInFlight) return sweepInFlight;
sweepInFlight = runSweep().finally(() => { sweepInFlight = null; });
return sweepInFlight;
}
async function runSweep(): Promise<SweepResult> {
if (!integrationsOn()) {
// Freeze pending rows as manual — they won't drain without integrations.
db()
@@ -49,11 +59,15 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea
return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
}
// Recovery frees capacity before admitting any more servers. This runs
// even without pending deployment rows (betas share the same slots).
await refreshRestartRollout();
const pending = db()
.prepare(
`SELECT deployment_id, server, attempts, restarted_at
FROM deployment_servers
WHERE restart_state IN ('pending','failed')`
`SELECT ds.deployment_id, ds.server, ds.attempts, ds.restarted_at, ds.files_changed
FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
WHERE ds.restart_state IN ('pending','failed')
ORDER BY d.started_at, d.id, ds.server`
)
.all() as PendingRow[];
if (pending.length === 0) {
@@ -65,6 +79,7 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea
// Split dedup caches by phase — the two phases talk to different endpoints.
const preRestart = new Map<string, Outcome>();
const postRestart = new Map<string, Outcome>();
const recoveringServers = new Set(pending.filter((row) => row.restarted_at).map((row) => row.server));
let checked = 0;
let restarted = 0;
@@ -80,13 +95,18 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea
);
for (const row of pending) {
// A failed file write isn't a restart request. Preserve the original
// failure until the operator repairs/retries that deployment.
if (row.files_changed === 0) continue;
const cache = row.restarted_at ? postRestart : preRestart;
let outcome = cache.get(row.server);
if (!outcome) {
checked++;
outcome = row.restarted_at
? await runHealthPhase(row.server)
: await runRestartPhase(row.server);
? runHealthPhase(row.server)
: recoveringServers.has(row.server)
? { state: 'pending', players: null, error: 'Queued behind an earlier restart of this server', restartedNow: false }
: await runRestartPhase(row.server);
cache.set(row.server, outcome);
if (outcome.state === 'done' && row.restarted_at) healthy++;
if (outcome.restartedNow) restarted++;
@@ -152,6 +172,8 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hea
// Phase 1: wait for the server to be empty, then Komodo-restart it.
async function runRestartPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
const waitReason = rolloutWaitReason(server);
if (waitReason) return { state: 'pending', players: null, error: waitReason, restartedNow: false };
const players = await rconGetPlayerCount(server);
if (players === null) {
return { state: 'pending', players: null, error: 'rcon unreachable', restartedNow: false };
@@ -159,32 +181,22 @@ async function runRestartPhase(server: string): Promise<{ state: 'done' | 'faile
if (players > 0) {
return { state: 'pending', players, error: null, restartedNow: false };
}
const r = await komodoRestart(server);
if (r.ok) {
const r = await restartInRollout(server);
if (r.kind === 'started') {
// Restart triggered — flip restarted_at so the next tick runs the
// health phase. Still 'pending' overall until health confirms.
return { state: 'pending', players: 0, error: null, restartedNow: true };
}
if (r.kind === 'no-name') {
return { state: 'failed', players: 0, error: r.message, restartedNow: false };
}
return { state: 'failed', players: 0, error: `komodo: ${r.message}`, restartedNow: false };
return { state: r.kind === 'queued' ? 'pending' : 'failed', players: 0, error: r.message, restartedNow: false };
}
// Phase 2: after Komodo-restart, wait until the container reports running AND
// RCON is answering. Only then does the deployment_server row transition to
// 'done'. If either check fails we stay pending and try again next tick.
async function runHealthPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
const h = await komodoInspectHealthy(server);
function runHealthPhase(server: string): { state: 'done' | 'pending'; players: number | null; error: string | null; restartedNow: boolean } {
const h = rolloutHealth(server);
if (!h.ok) {
// Include a truncated response snippet (when the extractor is the
// failure) so the operator can spot which Komodo field to teach.
const snippet = h.snippet ? ` snippet=${h.snippet.slice(0, 400)}` : '';
return { state: 'pending', players: null, error: `unhealthy: ${h.reason}${snippet}`, restartedNow: false };
return { state: 'pending', players: null, error: h.reason, restartedNow: false };
}
const rcon = await rconStatusResult(server);
if (!rcon.ok) {
return { state: 'pending', players: null, error: `rcon not answering: ${rcon.reason}`, restartedNow: false };
}
return { state: 'done', players: rcon.status.players, error: null, restartedNow: false };
return { state: 'done', players: h.players, error: null, restartedNow: false };
}