From edfe4dfc091d31df87a3d86eeea5c2fead48d45a Mon Sep 17 00:00:00 2001 From: josh Date: Sat, 12 Sep 2026 13:44:52 -0400 Subject: [PATCH] Add drift detection and removal as a real deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A drift deployment scans each server's sp/ tree for files not in the server_scripts inventory (untracked scripts left over from before EC was managing the fleet), deletes them, and records the removed bytes in deployment_files for audit. The existing sweep then runs the standard wait-for-empty → Komodo restart cycle on each affected server. To make that cycle safer, the sweep now has a second phase: after Komodo restart succeeds we wait for komodoInspectHealthy + RCON to both come back green before marking the deployment_server row 'done'. If either stays broken the row stays pending and keeps retrying. New: /api/drift (GET scan, POST run), drift panel on /deployments. Co-Authored-By: Claude Opus 4.7 --- src/lib/server/fleet/drift.ts | 206 ++++++++++++++++++++++++++++ src/lib/server/pipeline/sweep.ts | 124 ++++++++++------- src/routes/api/drift/+server.ts | 32 +++++ src/routes/deployments/+page.svelte | 125 ++++++++++++++++- tests/unit/sweep.test.ts | 2 +- 5 files changed, 440 insertions(+), 49 deletions(-) create mode 100644 src/lib/server/fleet/drift.ts create mode 100644 src/routes/api/drift/+server.ts diff --git a/src/lib/server/fleet/drift.ts b/src/lib/server/fleet/drift.ts new file mode 100644 index 0000000..e990e7a --- /dev/null +++ b/src/lib/server/fleet/drift.ts @@ -0,0 +1,206 @@ +import { readdir, readFile, rm, stat } from 'node:fs/promises'; +import { join, posix, relative, sep } from 'node:path'; +import { destOf } from '../config.js'; +import { db } from '../db/index.js'; +import { emit } from '../pipeline/bus.js'; +import { sha256Buffer } from './files.js'; +import { listServerScripts, removeScript } from './inventory.js'; +import { allServers, listServers } from './servers.js'; + +export interface DriftEntry { + server: string; + relPath: string; + size: number; + sha256: string; +} + +function toPosix(p: string): string { + return p.split(sep).join(posix.sep); +} + +async function walkFiles(root: string): Promise { + const out: string[] = []; + async function recur(dir: string): Promise { + let entries; + try { + entries = await readdir(dir, { withFileTypes: true }); + } catch { + return; + } + for (const e of entries) { + const abs = join(dir, e.name); + if (e.isDirectory()) await recur(abs); + else if (e.isFile()) out.push(toPosix(relative(root, abs))); + } + } + await recur(root); + return out.sort(); +} + +/** + * Scan one server's sp/ tree and return every file that isn't tracked in + * server_scripts. This includes user-placed scripts and leftovers from + * before Electric Cherry started tracking inventory. Beta artifacts + * (-beta.gsc, .OLD) are tracked while a beta is active so they don't + * appear as drift. + */ +export async function scanServerDrift(server: string): Promise { + const dest = destOf(server); + const onDisk = await walkFiles(dest); + if (onDisk.length === 0) return []; + const tracked = new Set(listServerScripts(server).map((r) => r.rel_path)); + + const drift: DriftEntry[] = []; + for (const relPath of onDisk) { + if (tracked.has(relPath)) continue; + const abs = join(dest, relPath); + try { + const st = await stat(abs); + const buf = await readFile(abs); + drift.push({ server, relPath, size: st.size, sha256: sha256Buffer(buf) }); + } catch { + /* file disappeared between walk and stat — skip */ + } + } + return drift; +} + +/** + * Scan the whole fleet. Servers are read from the DB (registered fleet), not + * the on-disk SERVERS_ROOT listing, so unregistered folders don't leak in. + */ +export async function scanFleetDrift(): Promise { + const servers = listServers().map((s) => s.name); + const registered = new Set(servers); + const allDirs = new Set(await allServers()); + // Only touch server dirs that are both registered AND present on disk. + const eligible = servers.filter((n) => registered.has(n) && allDirs.has(n)); + const out: DriftEntry[] = []; + for (const name of eligible) { + const rows = await scanServerDrift(name); + out.push(...rows); + } + return out; +} + +export interface RunDriftInput { + servers?: string[]; // if omitted, scans the whole fleet + dryRun?: boolean; + triggeredBy?: 'ui' | 'sweep' | 'cron'; +} + +export interface RunDriftResult { + deploymentId: string | null; + perServer: Array<{ server: string; removed: string[]; skipped: string[] }>; + totalRemoved: number; + totalServers: number; + dryRun: boolean; +} + +function deploymentIdNow(): string { + const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z'); + return `dep-${stamp}-drift`; +} + +/** + * Detect drift on the requested servers (or the whole fleet) and either + * report the plan (dryRun) or actually remove the untracked files as a + * single drift deployment. The sweep restarts each affected server once + * it's empty and waits for it to come back healthy. + */ +export async function runDriftDeployment(input: RunDriftInput = {}): Promise { + const { dryRun = false, triggeredBy = 'ui' } = input; + + const scanTargets = input.servers && input.servers.length > 0 + ? input.servers + : listServers().map((s) => s.name); + + const perServer: Array<{ server: string; removed: string[]; skipped: string[]; entries: DriftEntry[] }> = []; + for (const name of scanTargets) { + const entries = await scanServerDrift(name); + perServer.push({ server: name, removed: [], skipped: [], entries }); + } + + const totalDrift = perServer.reduce((sum, s) => sum + s.entries.length, 0); + if (totalDrift === 0 || dryRun) { + return { + deploymentId: null, + perServer: perServer.map((s) => ({ + server: s.server, + removed: s.entries.map((e) => e.relPath), + skipped: [] + })), + totalRemoved: totalDrift, + totalServers: perServer.filter((s) => s.entries.length > 0).length, + dryRun + }; + } + + // Commit. Create the deployment row FIRST so audit rows (which FK to it) + // can be inserted as we go. + const depId = deploymentIdNow(); + const now = new Date().toISOString(); + db().prepare( + `INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, ?)` + ).run(depId, now, triggeredBy); + const insertServer = db().prepare( + `INSERT INTO deployment_servers + (deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error) + VALUES (?, ?, ?, 'pending', NULL, NULL, NULL, 0, NULL)` + ); + const insertFile = db().prepare( + `INSERT INTO deployment_files + (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after) + VALUES (?, ?, 'removed', ?, ?, NULL, ?, NULL)` + ); + + const affected: string[] = []; + for (const s of perServer) { + if (s.entries.length === 0) continue; + const dest = destOf(s.server); + for (const entry of s.entries) { + const abs = join(dest, entry.relPath); + try { + const content = await readFile(abs); + await rm(abs, { force: true }); + removeScript(s.server, entry.relPath); + s.removed.push(entry.relPath); + insertFile.run(depId, s.server, entry.relPath, entry.sha256, content); + } catch (err) { + s.skipped.push(`${entry.relPath}: ${(err as Error).message}`); + } + } + if (s.removed.length > 0) { + insertServer.run(depId, s.server, s.removed.length); + affected.push(s.server); + } + } + + if (affected.length === 0) { + // Everything failed. Roll back the empty deployment row so it doesn't + // dangle in the UI as a mysterious no-op. + db().prepare('DELETE FROM deployments WHERE id = ?').run(depId); + return { + deploymentId: null, + perServer: perServer.map((s) => ({ server: s.server, removed: s.removed, skipped: s.skipped })), + totalRemoved: 0, + totalServers: 0, + dryRun: false + }; + } + + emit({ + type: 'deployment-opened', + deploymentId: depId, + action: 'drift', + servers: affected + }); + + return { + deploymentId: depId, + perServer: perServer.map((s) => ({ server: s.server, removed: s.removed, skipped: s.skipped })), + totalRemoved: perServer.reduce((sum, s) => sum + s.removed.length, 0), + totalServers: affected.length, + dryRun: false + }; +} diff --git a/src/lib/server/pipeline/sweep.ts b/src/lib/server/pipeline/sweep.ts index eb27047..53945fb 100644 --- a/src/lib/server/pipeline/sweep.ts +++ b/src/lib/server/pipeline/sweep.ts @@ -1,14 +1,15 @@ import { config, komodoConfigured, rconConfigured } from '../config.js'; import { db } from '../db/index.js'; import { emit } from './bus.js'; -import { rconGetPlayerCount } from '../integrations/rcon.js'; -import { komodoRestart } from '../integrations/komodo.js'; +import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js'; +import { komodoRestart, komodoInspectHealthy } from '../integrations/komodo.js'; import { setServerField } from '../fleet/servers.js'; interface PendingRow { deployment_id: string; server: string; attempts: number; + restarted_at: string | null; } function nowIso(): string { @@ -20,17 +21,23 @@ function integrationsOn(): boolean { } /** - * Re-check every pending/failed deployment_server row, restart the ones with - * zero players, hold the busy ones. Same-server dedup is per-sweep, so one - * restart resolves every pending row for that server. Ports `deploy_sweep` - * in push_scripts.sh:705 but talks RCON directly instead of routing through - * IW4MAdmin. + * Two-phase sweep of every pending/failed deployment_server row. + * + * Phase 1 (restarted_at IS NULL): the "wait for empty then restart" phase. + * - Poll RCON for player count. Busy or unreachable → stay pending. + * - Empty → call Komodo restart, set restarted_at, stay pending so the + * row picks up phase 2 on the next tick. + * + * Phase 2 (restarted_at IS NOT NULL): the "wait for healthy" phase. + * - Ask Komodo whether the container is running AND probe RCON. + * - Both green → done. Either red → stay pending; keep polling. + * + * Same-server dedup is per-tick per-phase so multiple concurrent deployments + * targeting the same server only cost one RCON+Komodo round trip per tick. */ -export async function sweep(): Promise<{ checked: number; restarted: number; held: number; failed: number }> { +export async function sweep(): Promise<{ checked: number; restarted: number; healthy: number; held: number; failed: number }> { if (!integrationsOn()) { - // Same shape as the shell script's line 714: mark rows manual so - // they don't hang forever waiting for something that will never - // arrive. + // Freeze pending rows as manual — they won't drain without integrations. db() .prepare( `UPDATE deployment_servers @@ -39,65 +46,53 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel ) .run(nowIso()); emit({ type: 'sweep-tick', frozen: false, reason: 'integrations not configured' }); - return { checked: 0, restarted: 0, held: 0, failed: 0 }; + return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 }; } const pending = db() .prepare( - `SELECT deployment_id, server, attempts + `SELECT deployment_id, server, attempts, restarted_at FROM deployment_servers WHERE restart_state IN ('pending','failed')` ) .all() as PendingRow[]; if (pending.length === 0) { emit({ type: 'sweep-tick', frozen: false }); - return { checked: 0, restarted: 0, held: 0, failed: 0 }; + return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 }; } - // Dedup across deployments: if we resolved server X once this tick, - // reuse the outcome for every other pending row on X. - type Outcome = { state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null }; - const seen = new Map(); + type Outcome = { state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }; + // Split dedup caches by phase — the two phases talk to different endpoints. + const preRestart = new Map(); + const postRestart = new Map(); let checked = 0; let restarted = 0; + let healthy = 0; let held = 0; let failed = 0; const applyOutcome = db().prepare( `UPDATE deployment_servers SET restart_state = ?, checked_at = ?, players = ?, error = ?, attempts = attempts + 1, - restarted_at = CASE WHEN ? = 'done' THEN ? ELSE restarted_at END + restarted_at = CASE WHEN ? = 1 THEN ? ELSE restarted_at END WHERE deployment_id = ? AND server = ?` ); for (const row of pending) { - let outcome = seen.get(row.server); + const cache = row.restarted_at ? postRestart : preRestart; + let outcome = cache.get(row.server); if (!outcome) { checked++; - const players = await rconGetPlayerCount(row.server); - if (players === null) { - outcome = { state: 'pending', players: null, error: 'rcon unreachable' }; - held++; - } else if (players > 0) { - outcome = { state: 'pending', players, error: null }; - held++; - } else { - const r = await komodoRestart(row.server); - if (r.ok) { - outcome = { state: 'done', players: 0, error: null }; - restarted++; - } else if (r.kind === 'no-name') { - outcome = { state: 'failed', players: 0, error: r.message }; - failed++; - } else { - outcome = { state: 'failed', players: 0, error: `komodo: ${r.message}` }; - failed++; - } - } - seen.set(row.server, outcome); - // Cache the last known player count on the server row so the - // dashboard can render a value between rcon polls. + outcome = row.restarted_at + ? await runHealthPhase(row.server) + : await runRestartPhase(row.server); + cache.set(row.server, outcome); + if (outcome.state === 'done' && row.restarted_at) healthy++; + if (outcome.restartedNow) restarted++; + if (outcome.state === 'pending') held++; + if (outcome.state === 'failed') failed++; + // Cache last known player count for the dashboard. if (outcome.players !== null) { setServerField(row.server, 'last_players', outcome.players); setServerField(row.server, 'last_checked_at', nowIso()); @@ -109,7 +104,7 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel now, outcome.players, outcome.error, - outcome.state, + outcome.restartedNow ? 1 : 0, now, row.deployment_id, row.server @@ -124,7 +119,7 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel }); } - // Finish deployments where every server is now non-pending + // Finish deployments where every server row is non-pending/non-failed. db() .prepare( `UPDATE deployments @@ -138,7 +133,6 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel ) .run(nowIso()); - // Prune rows older than DEPLOY_KEEP_DAYS that have no pending/failed servers const cutoff = new Date(Date.now() - config.deployKeepDays * 24 * 60 * 60 * 1000).toISOString(); db() .prepare( @@ -153,5 +147,41 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel .run(cutoff); emit({ type: 'sweep-tick', frozen: false }); - return { checked, restarted, held, failed }; + return { checked, restarted, healthy, held, failed }; +} + +// Phase 1: wait for the server to be empty, then Komodo-restart it. +async function runRestartPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> { + const players = await rconGetPlayerCount(server); + if (players === null) { + return { state: 'pending', players: null, error: 'rcon unreachable', restartedNow: false }; + } + if (players > 0) { + return { state: 'pending', players, error: null, restartedNow: false }; + } + const r = await komodoRestart(server); + if (r.ok) { + // Restart triggered — flip restarted_at so the next tick runs the + // health phase. Still 'pending' overall until health confirms. + return { state: 'pending', players: 0, error: null, restartedNow: true }; + } + if (r.kind === 'no-name') { + return { state: 'failed', players: 0, error: r.message, restartedNow: false }; + } + return { state: 'failed', players: 0, error: `komodo: ${r.message}`, restartedNow: false }; +} + +// Phase 2: after Komodo-restart, wait until the container reports running AND +// RCON is answering. Only then does the deployment_server row transition to +// 'done'. If either check fails we stay pending and try again next tick. +async function runHealthPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> { + const h = await komodoInspectHealthy(server); + if (!h.ok) { + return { state: 'pending', players: null, error: `unhealthy: ${h.reason}`, restartedNow: false }; + } + const rcon = await rconStatusResult(server); + if (!rcon.ok) { + return { state: 'pending', players: null, error: `rcon not answering: ${rcon.reason}`, restartedNow: false }; + } + return { state: 'done', players: rcon.status.players, error: null, restartedNow: false }; } diff --git a/src/routes/api/drift/+server.ts b/src/routes/api/drift/+server.ts new file mode 100644 index 0000000..abbe005 --- /dev/null +++ b/src/routes/api/drift/+server.ts @@ -0,0 +1,32 @@ +import { json } from '@sveltejs/kit'; +import { runDriftDeployment, scanFleetDrift } from '$lib/server/fleet/drift.js'; +import { kick } from '$lib/server/pipeline/worker.js'; + +// GET returns the current drift plan for the whole fleet without touching disk. +export async function GET() { + const entries = await scanFleetDrift(); + const byServer = new Map>(); + for (const e of entries) { + const arr = byServer.get(e.server) ?? []; + arr.push({ relPath: e.relPath, size: e.size, sha256: e.sha256 }); + byServer.set(e.server, arr); + } + return json({ + totalDrift: entries.length, + servers: [...byServer.entries()].map(([server, files]) => ({ server, files })) + }); +} + +// POST removes drift and creates a drift deployment. Body: { servers?: string[], dryRun?: boolean } +export async function POST({ request }: { request: Request }) { + const body = (await request + .json() + .catch(() => ({}))) as { servers?: string[]; dryRun?: boolean }; + const result = await runDriftDeployment({ + servers: body.servers, + dryRun: body.dryRun ?? false, + triggeredBy: 'ui' + }); + if (result.deploymentId) void kick(); + return json({ ok: true, result }); +} diff --git a/src/routes/deployments/+page.svelte b/src/routes/deployments/+page.svelte index 95c20d5..e5ba745 100644 --- a/src/routes/deployments/+page.svelte +++ b/src/routes/deployments/+page.svelte @@ -17,13 +17,59 @@ servers: ServerRow[]; } + interface DriftFile { relPath: string; size: number; sha256: string } + interface DriftServer { server: string; files: DriftFile[] } + interface DriftPreview { totalDrift: number; servers: DriftServer[] } + let deployments = $state([]); let loading = $state(true); + let drift = $state(null); + let scanning = $state(false); + let running = $state(false); + let flash = $state(null); - onMount(async () => { + async function loadDeployments(): Promise { const res = await fetch('/api/deployments'); const body = (await res.json()) as { deployments: Deployment[] }; deployments = body.deployments; + } + + async function scanDrift(): Promise { + scanning = true; + try { + const res = await fetch('/api/drift'); + drift = (await res.json()) as DriftPreview; + } finally { + scanning = false; + } + } + + async function runDrift(): Promise { + if (!drift || drift.totalDrift === 0) return; + if (!confirm(`Remove ${drift.totalDrift} untracked file(s) across ${drift.servers.length} server(s)?`)) return; + running = true; + flash = null; + try { + const res = await fetch('/api/drift', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({}) + }); + const body = (await res.json()) as { + ok: boolean; + error?: string; + result?: { deploymentId: string | null; totalRemoved: number; totalServers: number }; + }; + if (!body.ok) { flash = body.error ?? 'drift run failed'; return; } + flash = `removed ${body.result?.totalRemoved ?? 0} files across ${body.result?.totalServers ?? 0} servers`; + await Promise.all([loadDeployments(), scanDrift()]); + } finally { + running = false; + } + } + + onMount(async () => { + await Promise.all([loadDeployments(), scanDrift()]); loading = false; }); @@ -52,6 +98,35 @@ {#if loading}

loading…

{:else} +
+
+

Drift{drift ? ` · ${drift.totalDrift} untracked file${drift.totalDrift === 1 ? '' : 's'}` : ''}

+
+ + +
+
+ {#if drift === null} +

scan hasn't run yet

+ {:else if drift.totalDrift === 0} +

inventory matches disk — no drift

+ {:else} + {#each drift.servers as s (s.server)} +
+
{s.server} · {s.files.length} file{s.files.length === 1 ? '' : 's'}
+
+ {#each s.files as f (f.relPath)} + {f.relPath} + {/each} +
+
+ {/each} + {/if} + {#if flash}

{flash}

{/if} +
+

Active

{#if active.length === 0} @@ -168,4 +243,52 @@ font-family: var(--font-mono); font-size: var(--text-base); } + + .drift-panel { + background: var(--bg-nav); + border: 1px solid var(--border); + border-radius: var(--r-md); + padding: var(--pad-panel); + margin-bottom: var(--gap-section); + } + .drift-head { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 14px; } + .drift-head h2 { margin: 0; } + .drift-actions { display: flex; gap: 8px; } + .drift-actions button { + background: var(--bg-card); + color: var(--ink-100); + border: 1px solid var(--border); + border-radius: var(--r-sm); + padding: 8px 16px; + font-family: var(--font-body); + font-size: var(--text-sm); + cursor: pointer; + } + .drift-actions button:hover:not(:disabled) { border-color: var(--accent); box-shadow: var(--accent-glow); } + .drift-actions button.primary { + background: linear-gradient(180deg, var(--danger-500), #a01e39); + border-color: rgba(255, 48, 80, 0.6); + color: var(--cream); + font-weight: 600; + } + .drift-actions button:disabled { opacity: 0.4; cursor: default; } + .drift-server { + background: var(--bg-card); + border: 1px solid var(--border); + border-radius: var(--r-sm); + padding: 10px 14px; + margin: 8px 0; + } + .drift-server-head { color: var(--cream); font-size: var(--text-mono); margin-bottom: 6px; } + .drift-files { display: flex; flex-wrap: wrap; gap: 6px; } + .drift-file { + background: rgba(255, 48, 80, 0.08); + border: 1px solid rgba(255, 48, 80, 0.25); + color: var(--danger-500); + padding: 3px 8px; + border-radius: var(--r-sm); + font-size: var(--text-mono-sm); + } + .flash { color: var(--spark-400); font-family: var(--font-mono); font-size: var(--text-mono); margin-top: 12px; } + .mono { font-family: var(--font-mono); } diff --git a/tests/unit/sweep.test.ts b/tests/unit/sweep.test.ts index 30c7ef2..59a32cd 100644 --- a/tests/unit/sweep.test.ts +++ b/tests/unit/sweep.test.ts @@ -53,6 +53,6 @@ describe('sweep', () => { await seedServerRegistry(); const result = await sweep(); - expect(result).toEqual({ checked: 0, restarted: 0, held: 0, failed: 0 }); + expect(result).toEqual({ checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 }); }); });