josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: make provisioning own instance lifecycle operations

Browse Source
This commit is contained in:
josh
2026-09-27 14:45:00 -04:00
parent cfed584614
commit df265d1900
44 changed files with 1791 additions and 1609 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/sweep.ts
+31 -184
View File
@@ -1,202 +1,49 @@
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { config } from '../config.js';
import { db } from '../db/index.js';
import { emit } from './bus.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { setServerField } from '../fleet/servers.js';
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
import { requestRestart } from '../provisioning/index.js';
interface PendingRow {
deployment_id: string;
server: string;
attempts: number;
restarted_at: string | null;
files_changed: number;
deployment_id: string; server: string; restart_state: string;
restarted_at: string | null; files_changed: number;
}
type SweepResult = { checked: number; restarted: number; healthy: number; held: number; failed: number };
let sweepInFlight: Promise<SweepResult> | null = null;
const nowIso = () => new Date().toISOString();
function nowIso(): string {
return new Date().toISOString();
}
function integrationsOn(): boolean {
return komodoConfigured() && rconConfigured();
}
/**
* Two-phase sweep of every pending/failed deployment_server row.
*
* Phase 1 (restarted_at IS NULL): the "wait for empty then restart" phase.
* - Poll RCON for player count. Busy or unreachable → stay pending.
* - Empty → call Komodo restart, set restarted_at, stay pending so the
* row picks up phase 2 on the next tick.
*
* Phase 2 (restarted_at IS NOT NULL): the "wait for healthy" phase.
* - Ask Komodo whether the container is running AND probe RCON.
* - Both green → done. Either red → stay pending; keep polling.
*
* Same-server dedup is per-tick per-phase so multiple concurrent deployments
* targeting the same server only cost one RCON+Komodo round trip per tick.
*/
/** Submit activation requests and mirror their results. Provisioning owns all execution. */
export function sweep(): Promise<SweepResult> {
if (sweepInFlight) return sweepInFlight;
sweepInFlight = runSweep().finally(() => { sweepInFlight = null; });
return sweepInFlight;
}
async function runSweep(): Promise<SweepResult> {
if (!integrationsOn()) {
// Freeze pending rows as manual — they won't drain without integrations.
db()
.prepare(
`UPDATE deployment_servers
SET restart_state = 'manual', checked_at = ?, error = 'integrations not configured'
WHERE restart_state IN ('pending','failed')`
)
.run(nowIso());
emit({ type: 'sweep-tick', frozen: false, reason: 'integrations not configured' });
return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
}
// Recovery frees capacity before admitting any more servers. This runs
// even without pending deployment rows (betas share the same slots).
await refreshRestartRollout();
const pending = db()
.prepare(
`SELECT ds.deployment_id, ds.server, ds.attempts, ds.restarted_at, ds.files_changed
FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
WHERE ds.restart_state IN ('pending','failed')
ORDER BY d.started_at, d.id, ds.server`
)
.all() as PendingRow[];
if (pending.length === 0) {
emit({ type: 'sweep-tick', frozen: false });
return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
}
type Outcome = { state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean };
// Split dedup caches by phase — the two phases talk to different endpoints.
const preRestart = new Map<string, Outcome>();
const postRestart = new Map<string, Outcome>();
const recoveringServers = new Set(pending.filter((row) => row.restarted_at).map((row) => row.server));
let checked = 0;
let restarted = 0;
let healthy = 0;
let held = 0;
let failed = 0;
const applyOutcome = db().prepare(
`UPDATE deployment_servers
SET restart_state = ?, checked_at = ?, players = ?, error = ?, attempts = attempts + 1,
restarted_at = CASE WHEN ? = 1 THEN ? ELSE restarted_at END
WHERE deployment_id = ? AND server = ?`
);
const pending = db().prepare(`SELECT ds.* FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
WHERE ds.restart_state IN ('pending','failed') ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[];
const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
for (const row of pending) {
// A failed file write isn't a restart request. Preserve the original
// failure until the operator repairs/retries that deployment.
if (row.files_changed === 0) continue;
const cache = row.restarted_at ? postRestart : preRestart;
let outcome = cache.get(row.server);
if (!outcome) {
checked++;
outcome = row.restarted_at
? runHealthPhase(row.server)
: recoveringServers.has(row.server)
? { state: 'pending', players: null, error: 'Queued behind an earlier restart of this server', restartedNow: false }
: await runRestartPhase(row.server);
cache.set(row.server, outcome);
if (outcome.state === 'done' && row.restarted_at) healthy++;
if (outcome.restartedNow) restarted++;
if (outcome.state === 'pending') held++;
if (outcome.state === 'failed') failed++;
// Cache last known player count for the dashboard.
if (outcome.players !== null) {
setServerField(row.server, 'last_players', outcome.players);
setServerField(row.server, 'last_checked_at', nowIso());
}
}
const now = nowIso();
applyOutcome.run(
outcome.state,
now,
outcome.players,
outcome.error,
outcome.restartedNow ? 1 : 0,
now,
row.deployment_id,
row.server
);
emit({
type: 'server-state',
deploymentId: row.deployment_id,
server: row.server,
state: outcome.state,
players: outcome.players,
error: outcome.error
});
if (!row.files_changed) continue; // A failed file write is not a restart request.
const operation = requestRestart(row.server, { kind: 'campaign', id: row.deployment_id });
result.checked++;
const state = operation.state === 'done' ? 'done'
: ['failed','interrupted','cancelled'].includes(operation.state) ? 'failed' : 'pending';
const restartedAt = ['awaiting-health','done'].includes(operation.state) ? operation.submitted_at : row.restarted_at;
if (restartedAt && !row.restarted_at) result.restarted++;
if (state === 'done') result.healthy++;
if (state === 'pending') result.held++;
if (state === 'failed') result.failed++;
const error = state === 'done' ? null : operation.error ?? operation.stage;
db().prepare(`UPDATE deployment_servers SET restart_state = ?, checked_at = ?, players = ?, error = ?,
attempts = attempts + CASE WHEN ? IS NOT NULL AND checked_at IS NOT ? THEN 1 ELSE 0 END,
restarted_at = COALESCE(restarted_at, ?) WHERE deployment_id = ? AND server = ?`)
.run(state, operation.checked_at, operation.players, error, operation.checked_at, operation.checked_at, restartedAt, row.deployment_id, row.server);
emit({ type: 'server-state', deploymentId: row.deployment_id, server: row.server, state, players: operation.players, error });
}
// Finish deployments where every server row is non-pending/non-failed.
db()
.prepare(
`UPDATE deployments
SET finished_at = ?
WHERE finished_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM deployment_servers
WHERE deployment_id = deployments.id
AND restart_state IN ('pending','failed')
)`
)
.run(nowIso());
const cutoff = new Date(Date.now() - config.deployKeepDays * 24 * 60 * 60 * 1000).toISOString();
db()
.prepare(
`DELETE FROM deployments
WHERE started_at < ?
AND NOT EXISTS (
SELECT 1 FROM deployment_servers
WHERE deployment_id = deployments.id
AND restart_state IN ('pending','failed')
)`
)
.run(cutoff);
db().prepare(`UPDATE deployments SET finished_at = ? WHERE finished_at IS NULL AND NOT EXISTS (
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND restart_state IN ('pending','failed','held'))`).run(nowIso());
db().prepare(`DELETE FROM deployments WHERE started_at < ? AND NOT EXISTS (
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND restart_state IN ('pending','failed','held'))`)
.run(new Date(Date.now() - config.deployKeepDays * 86_400_000).toISOString());
emit({ type: 'sweep-tick', frozen: false });
return { checked, restarted, healthy, held, failed };
}
// Phase 1: wait for the server to be empty, then Komodo-restart it.
async function runRestartPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
const waitReason = rolloutWaitReason(server);
if (waitReason) return { state: 'pending', players: null, error: waitReason, restartedNow: false };
const players = await rconGetPlayerCount(server);
if (players === null) {
return { state: 'pending', players: null, error: 'rcon unreachable', restartedNow: false };
}
if (players > 0) {
return { state: 'pending', players, error: null, restartedNow: false };
}
const r = await restartInRollout(server);
if (r.kind === 'started') {
// Restart triggered — flip restarted_at so the next tick runs the
// health phase. Still 'pending' overall until health confirms.
return { state: 'pending', players: 0, error: null, restartedNow: true };
}
return { state: r.kind === 'queued' ? 'pending' : 'failed', players: 0, error: r.message, restartedNow: false };
}
// Phase 2: after Komodo-restart, wait until the container reports running AND
// RCON is answering. Only then does the deployment_server row transition to
// 'done'. If either check fails we stay pending and try again next tick.
function runHealthPhase(server: string): { state: 'done' | 'pending'; players: number | null; error: string | null; restartedNow: boolean } {
const h = rolloutHealth(server);
if (!h.ok) {
return { state: 'pending', players: null, error: h.reason, restartedNow: false };
}
return { state: 'done', players: h.players, error: null, restartedNow: false };
return result;
}