josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add drift detection and removal as a real deployment

Browse Source 
A drift deployment scans each server's sp/ tree for files not in the
server_scripts inventory (untracked scripts left over from before EC was
managing the fleet), deletes them, and records the removed bytes in
deployment_files for audit. The existing sweep then runs the standard
wait-for-empty → Komodo restart cycle on each affected server.

To make that cycle safer, the sweep now has a second phase: after Komodo
restart succeeds we wait for komodoInspectHealthy + RCON to both come
back green before marking the deployment_server row 'done'. If either
stays broken the row stays pending and keeps retrying.

New: /api/drift (GET scan, POST run), drift panel on /deployments.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
josh
2026-09-12 13:44:52 -04:00
co-authored by Claude Opus 4.7
parent 590f803d93
commit edfe4dfc09
5 changed files with 440 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/sweep.ts
+77 -47
View File
@@ -1,14 +1,15 @@
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { db } from '../db/index.js';
import { emit } from './bus.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { komodoRestart } from '../integrations/komodo.js';
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
import { komodoRestart, komodoInspectHealthy } from '../integrations/komodo.js';
import { setServerField } from '../fleet/servers.js';
interface PendingRow {
deployment_id: string;
server: string;
attempts: number;
restarted_at: string | null;
}
function nowIso(): string {
@@ -20,17 +21,23 @@ function integrationsOn(): boolean {
}
/**
* Re-check every pending/failed deployment_server row, restart the ones with
* zero players, hold the busy ones. Same-server dedup is per-sweep, so one
* restart resolves every pending row for that server. Ports `deploy_sweep`
* in push_scripts.sh:705 but talks RCON directly instead of routing through
* IW4MAdmin.
* Two-phase sweep of every pending/failed deployment_server row.
*
* Phase 1 (restarted_at IS NULL): the "wait for empty then restart" phase.
* - Poll RCON for player count. Busy or unreachable → stay pending.
* - Empty → call Komodo restart, set restarted_at, stay pending so the
* row picks up phase 2 on the next tick.
*
* Phase 2 (restarted_at IS NOT NULL): the "wait for healthy" phase.
* - Ask Komodo whether the container is running AND probe RCON.
* - Both green → done. Either red → stay pending; keep polling.
*
* Same-server dedup is per-tick per-phase so multiple concurrent deployments
* targeting the same server only cost one RCON+Komodo round trip per tick.
*/
export async function sweep(): Promise<{ checked: number; restarted: number; held: number; failed: number }> {
export async function sweep(): Promise<{ checked: number; restarted: number; healthy: number; held: number; failed: number }> {
if (!integrationsOn()) {
// Same shape as the shell script's line 714: mark rows manual so
// they don't hang forever waiting for something that will never
// arrive.
// Freeze pending rows as manual — they won't drain without integrations.
db()
.prepare(
`UPDATE deployment_servers
@@ -39,65 +46,53 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
)
.run(nowIso());
emit({ type: 'sweep-tick', frozen: false, reason: 'integrations not configured' });
return { checked: 0, restarted: 0, held: 0, failed: 0 };
return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
}
const pending = db()
.prepare(
`SELECT deployment_id, server, attempts
`SELECT deployment_id, server, attempts, restarted_at
FROM deployment_servers
WHERE restart_state IN ('pending','failed')`
)
.all() as PendingRow[];
if (pending.length === 0) {
emit({ type: 'sweep-tick', frozen: false });
return { checked: 0, restarted: 0, held: 0, failed: 0 };
return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
}
// Dedup across deployments: if we resolved server X once this tick,
// reuse the outcome for every other pending row on X.
type Outcome = { state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null };
const seen = new Map<string, Outcome>();
type Outcome = { state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean };
// Split dedup caches by phase — the two phases talk to different endpoints.
const preRestart = new Map<string, Outcome>();
const postRestart = new Map<string, Outcome>();
let checked = 0;
let restarted = 0;
let healthy = 0;
let held = 0;
let failed = 0;
const applyOutcome = db().prepare(
`UPDATE deployment_servers
SET restart_state = ?, checked_at = ?, players = ?, error = ?, attempts = attempts + 1,
restarted_at = CASE WHEN ? = 'done' THEN ? ELSE restarted_at END
restarted_at = CASE WHEN ? = 1 THEN ? ELSE restarted_at END
WHERE deployment_id = ? AND server = ?`
);
for (const row of pending) {
let outcome = seen.get(row.server);
const cache = row.restarted_at ? postRestart : preRestart;
let outcome = cache.get(row.server);
if (!outcome) {
checked++;
const players = await rconGetPlayerCount(row.server);
if (players === null) {
outcome = { state: 'pending', players: null, error: 'rcon unreachable' };
held++;
} else if (players > 0) {
outcome = { state: 'pending', players, error: null };
held++;
} else {
const r = await komodoRestart(row.server);
if (r.ok) {
outcome = { state: 'done', players: 0, error: null };
restarted++;
} else if (r.kind === 'no-name') {
outcome = { state: 'failed', players: 0, error: r.message };
failed++;
} else {
outcome = { state: 'failed', players: 0, error: `komodo: ${r.message}` };
failed++;
}
}
seen.set(row.server, outcome);
// Cache the last known player count on the server row so the
// dashboard can render a value between rcon polls.
outcome = row.restarted_at
? await runHealthPhase(row.server)
: await runRestartPhase(row.server);
cache.set(row.server, outcome);
if (outcome.state === 'done' && row.restarted_at) healthy++;
if (outcome.restartedNow) restarted++;
if (outcome.state === 'pending') held++;
if (outcome.state === 'failed') failed++;
// Cache last known player count for the dashboard.
if (outcome.players !== null) {
setServerField(row.server, 'last_players', outcome.players);
setServerField(row.server, 'last_checked_at', nowIso());
@@ -109,7 +104,7 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
now,
outcome.players,
outcome.error,
outcome.state,
outcome.restartedNow ? 1 : 0,
now,
row.deployment_id,
row.server
@@ -124,7 +119,7 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
});
}
// Finish deployments where every server is now non-pending
// Finish deployments where every server row is non-pending/non-failed.
db()
.prepare(
`UPDATE deployments
@@ -138,7 +133,6 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
)
.run(nowIso());
// Prune rows older than DEPLOY_KEEP_DAYS that have no pending/failed servers
const cutoff = new Date(Date.now() - config.deployKeepDays * 24 * 60 * 60 * 1000).toISOString();
db()
.prepare(
@@ -153,5 +147,41 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
.run(cutoff);
emit({ type: 'sweep-tick', frozen: false });
return { checked, restarted, held, failed };
return { checked, restarted, healthy, held, failed };
}
// Phase 1: wait for the server to be empty, then Komodo-restart it.
async function runRestartPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
const players = await rconGetPlayerCount(server);
if (players === null) {
return { state: 'pending', players: null, error: 'rcon unreachable', restartedNow: false };
}
if (players > 0) {
return { state: 'pending', players, error: null, restartedNow: false };
}
const r = await komodoRestart(server);
if (r.ok) {
// Restart triggered — flip restarted_at so the next tick runs the
// health phase. Still 'pending' overall until health confirms.
return { state: 'pending', players: 0, error: null, restartedNow: true };
}
if (r.kind === 'no-name') {
return { state: 'failed', players: 0, error: r.message, restartedNow: false };
}
return { state: 'failed', players: 0, error: `komodo: ${r.message}`, restartedNow: false };
}
// Phase 2: after Komodo-restart, wait until the container reports running AND
// RCON is answering. Only then does the deployment_server row transition to
// 'done'. If either check fails we stay pending and try again next tick.
async function runHealthPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
const h = await komodoInspectHealthy(server);
if (!h.ok) {
return { state: 'pending', players: null, error: `unhealthy: ${h.reason}`, restartedNow: false };
}
const rcon = await rconStatusResult(server);
if (!rcon.ok) {
return { state: 'pending', players: null, error: `rcon not answering: ${rcon.reason}`, restartedNow: false };
}
return { state: 'done', players: rcon.status.players, error: null, restartedNow: false };
}