Add drift detection and removal as a real deployment

A drift deployment scans each server's sp/ tree for files not in the
server_scripts inventory (untracked scripts left over from before EC was
managing the fleet), deletes them, and records the removed bytes in
deployment_files for audit. The existing sweep then runs the standard
wait-for-empty → Komodo restart cycle on each affected server.

To make that cycle safer, the sweep now has a second phase: after Komodo
restart succeeds we wait for komodoInspectHealthy + RCON to both come
back green before marking the deployment_server row 'done'. If either
stays broken the row stays pending and keeps retrying.

New: /api/drift (GET scan, POST run), drift panel on /deployments.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-09-12 13:44:52 -04:00
co-authored by Claude Opus 4.7
parent 590f803d93
commit edfe4dfc09
5 changed files with 440 additions and 49 deletions
+206
View File
@@ -0,0 +1,206 @@
import { readdir, readFile, rm, stat } from 'node:fs/promises';
import { join, posix, relative, sep } from 'node:path';
import { destOf } from '../config.js';
import { db } from '../db/index.js';
import { emit } from '../pipeline/bus.js';
import { sha256Buffer } from './files.js';
import { listServerScripts, removeScript } from './inventory.js';
import { allServers, listServers } from './servers.js';
export interface DriftEntry {
server: string;
relPath: string;
size: number;
sha256: string;
}
function toPosix(p: string): string {
return p.split(sep).join(posix.sep);
}
async function walkFiles(root: string): Promise<string[]> {
const out: string[] = [];
async function recur(dir: string): Promise<void> {
let entries;
try {
entries = await readdir(dir, { withFileTypes: true });
} catch {
return;
}
for (const e of entries) {
const abs = join(dir, e.name);
if (e.isDirectory()) await recur(abs);
else if (e.isFile()) out.push(toPosix(relative(root, abs)));
}
}
await recur(root);
return out.sort();
}
/**
* Scan one server's sp/ tree and return every file that isn't tracked in
* server_scripts. This includes user-placed scripts and leftovers from
* before Electric Cherry started tracking inventory. Beta artifacts
* (-beta.gsc, .OLD) are tracked while a beta is active so they don't
* appear as drift.
*/
export async function scanServerDrift(server: string): Promise<DriftEntry[]> {
const dest = destOf(server);
const onDisk = await walkFiles(dest);
if (onDisk.length === 0) return [];
const tracked = new Set(listServerScripts(server).map((r) => r.rel_path));
const drift: DriftEntry[] = [];
for (const relPath of onDisk) {
if (tracked.has(relPath)) continue;
const abs = join(dest, relPath);
try {
const st = await stat(abs);
const buf = await readFile(abs);
drift.push({ server, relPath, size: st.size, sha256: sha256Buffer(buf) });
} catch {
/* file disappeared between walk and stat — skip */
}
}
return drift;
}
/**
* Scan the whole fleet. Servers are read from the DB (registered fleet), not
* the on-disk SERVERS_ROOT listing, so unregistered folders don't leak in.
*/
export async function scanFleetDrift(): Promise<DriftEntry[]> {
const servers = listServers().map((s) => s.name);
const registered = new Set(servers);
const allDirs = new Set(await allServers());
// Only touch server dirs that are both registered AND present on disk.
const eligible = servers.filter((n) => registered.has(n) && allDirs.has(n));
const out: DriftEntry[] = [];
for (const name of eligible) {
const rows = await scanServerDrift(name);
out.push(...rows);
}
return out;
}
export interface RunDriftInput {
servers?: string[]; // if omitted, scans the whole fleet
dryRun?: boolean;
triggeredBy?: 'ui' | 'sweep' | 'cron';
}
export interface RunDriftResult {
deploymentId: string | null;
perServer: Array<{ server: string; removed: string[]; skipped: string[] }>;
totalRemoved: number;
totalServers: number;
dryRun: boolean;
}
function deploymentIdNow(): string {
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
return `dep-${stamp}-drift`;
}
/**
* Detect drift on the requested servers (or the whole fleet) and either
* report the plan (dryRun) or actually remove the untracked files as a
* single drift deployment. The sweep restarts each affected server once
* it's empty and waits for it to come back healthy.
*/
export async function runDriftDeployment(input: RunDriftInput = {}): Promise<RunDriftResult> {
const { dryRun = false, triggeredBy = 'ui' } = input;
const scanTargets = input.servers && input.servers.length > 0
? input.servers
: listServers().map((s) => s.name);
const perServer: Array<{ server: string; removed: string[]; skipped: string[]; entries: DriftEntry[] }> = [];
for (const name of scanTargets) {
const entries = await scanServerDrift(name);
perServer.push({ server: name, removed: [], skipped: [], entries });
}
const totalDrift = perServer.reduce((sum, s) => sum + s.entries.length, 0);
if (totalDrift === 0 || dryRun) {
return {
deploymentId: null,
perServer: perServer.map((s) => ({
server: s.server,
removed: s.entries.map((e) => e.relPath),
skipped: []
})),
totalRemoved: totalDrift,
totalServers: perServer.filter((s) => s.entries.length > 0).length,
dryRun
};
}
// Commit. Create the deployment row FIRST so audit rows (which FK to it)
// can be inserted as we go.
const depId = deploymentIdNow();
const now = new Date().toISOString();
db().prepare(
`INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, ?)`
).run(depId, now, triggeredBy);
const insertServer = db().prepare(
`INSERT INTO deployment_servers
(deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error)
VALUES (?, ?, ?, 'pending', NULL, NULL, NULL, 0, NULL)`
);
const insertFile = db().prepare(
`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
VALUES (?, ?, 'removed', ?, ?, NULL, ?, NULL)`
);
const affected: string[] = [];
for (const s of perServer) {
if (s.entries.length === 0) continue;
const dest = destOf(s.server);
for (const entry of s.entries) {
const abs = join(dest, entry.relPath);
try {
const content = await readFile(abs);
await rm(abs, { force: true });
removeScript(s.server, entry.relPath);
s.removed.push(entry.relPath);
insertFile.run(depId, s.server, entry.relPath, entry.sha256, content);
} catch (err) {
s.skipped.push(`${entry.relPath}: ${(err as Error).message}`);
}
}
if (s.removed.length > 0) {
insertServer.run(depId, s.server, s.removed.length);
affected.push(s.server);
}
}
if (affected.length === 0) {
// Everything failed. Roll back the empty deployment row so it doesn't
// dangle in the UI as a mysterious no-op.
db().prepare('DELETE FROM deployments WHERE id = ?').run(depId);
return {
deploymentId: null,
perServer: perServer.map((s) => ({ server: s.server, removed: s.removed, skipped: s.skipped })),
totalRemoved: 0,
totalServers: 0,
dryRun: false
};
}
emit({
type: 'deployment-opened',
deploymentId: depId,
action: 'drift',
servers: affected
});
return {
deploymentId: depId,
perServer: perServer.map((s) => ({ server: s.server, removed: s.removed, skipped: s.skipped })),
totalRemoved: perServer.reduce((sum, s) => sum + s.removed.length, 0),
totalServers: affected.length,
dryRun: false
};
}
+77 -47
View File
@@ -1,14 +1,15 @@
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { db } from '../db/index.js';
import { emit } from './bus.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { komodoRestart } from '../integrations/komodo.js';
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
import { komodoRestart, komodoInspectHealthy } from '../integrations/komodo.js';
import { setServerField } from '../fleet/servers.js';
interface PendingRow {
deployment_id: string;
server: string;
attempts: number;
restarted_at: string | null;
}
function nowIso(): string {
@@ -20,17 +21,23 @@ function integrationsOn(): boolean {
}
/**
* Re-check every pending/failed deployment_server row, restart the ones with
* zero players, hold the busy ones. Same-server dedup is per-sweep, so one
* restart resolves every pending row for that server. Ports `deploy_sweep`
* in push_scripts.sh:705 but talks RCON directly instead of routing through
* IW4MAdmin.
* Two-phase sweep of every pending/failed deployment_server row.
*
* Phase 1 (restarted_at IS NULL): the "wait for empty then restart" phase.
* - Poll RCON for player count. Busy or unreachable → stay pending.
* - Empty → call Komodo restart, set restarted_at, stay pending so the
* row picks up phase 2 on the next tick.
*
* Phase 2 (restarted_at IS NOT NULL): the "wait for healthy" phase.
* - Ask Komodo whether the container is running AND probe RCON.
* - Both green → done. Either red → stay pending; keep polling.
*
* Same-server dedup is per-tick per-phase so multiple concurrent deployments
* targeting the same server only cost one RCON+Komodo round trip per tick.
*/
export async function sweep(): Promise<{ checked: number; restarted: number; held: number; failed: number }> {
export async function sweep(): Promise<{ checked: number; restarted: number; healthy: number; held: number; failed: number }> {
if (!integrationsOn()) {
// Same shape as the shell script's line 714: mark rows manual so
// they don't hang forever waiting for something that will never
// arrive.
// Freeze pending rows as manual — they won't drain without integrations.
db()
.prepare(
`UPDATE deployment_servers
@@ -39,65 +46,53 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
)
.run(nowIso());
emit({ type: 'sweep-tick', frozen: false, reason: 'integrations not configured' });
return { checked: 0, restarted: 0, held: 0, failed: 0 };
return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
}
const pending = db()
.prepare(
`SELECT deployment_id, server, attempts
`SELECT deployment_id, server, attempts, restarted_at
FROM deployment_servers
WHERE restart_state IN ('pending','failed')`
)
.all() as PendingRow[];
if (pending.length === 0) {
emit({ type: 'sweep-tick', frozen: false });
return { checked: 0, restarted: 0, held: 0, failed: 0 };
return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
}
// Dedup across deployments: if we resolved server X once this tick,
// reuse the outcome for every other pending row on X.
type Outcome = { state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null };
const seen = new Map<string, Outcome>();
type Outcome = { state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean };
// Split dedup caches by phase — the two phases talk to different endpoints.
const preRestart = new Map<string, Outcome>();
const postRestart = new Map<string, Outcome>();
let checked = 0;
let restarted = 0;
let healthy = 0;
let held = 0;
let failed = 0;
const applyOutcome = db().prepare(
`UPDATE deployment_servers
SET restart_state = ?, checked_at = ?, players = ?, error = ?, attempts = attempts + 1,
restarted_at = CASE WHEN ? = 'done' THEN ? ELSE restarted_at END
restarted_at = CASE WHEN ? = 1 THEN ? ELSE restarted_at END
WHERE deployment_id = ? AND server = ?`
);
for (const row of pending) {
let outcome = seen.get(row.server);
const cache = row.restarted_at ? postRestart : preRestart;
let outcome = cache.get(row.server);
if (!outcome) {
checked++;
const players = await rconGetPlayerCount(row.server);
if (players === null) {
outcome = { state: 'pending', players: null, error: 'rcon unreachable' };
held++;
} else if (players > 0) {
outcome = { state: 'pending', players, error: null };
held++;
} else {
const r = await komodoRestart(row.server);
if (r.ok) {
outcome = { state: 'done', players: 0, error: null };
restarted++;
} else if (r.kind === 'no-name') {
outcome = { state: 'failed', players: 0, error: r.message };
failed++;
} else {
outcome = { state: 'failed', players: 0, error: `komodo: ${r.message}` };
failed++;
}
}
seen.set(row.server, outcome);
// Cache the last known player count on the server row so the
// dashboard can render a value between rcon polls.
outcome = row.restarted_at
? await runHealthPhase(row.server)
: await runRestartPhase(row.server);
cache.set(row.server, outcome);
if (outcome.state === 'done' && row.restarted_at) healthy++;
if (outcome.restartedNow) restarted++;
if (outcome.state === 'pending') held++;
if (outcome.state === 'failed') failed++;
// Cache last known player count for the dashboard.
if (outcome.players !== null) {
setServerField(row.server, 'last_players', outcome.players);
setServerField(row.server, 'last_checked_at', nowIso());
@@ -109,7 +104,7 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
now,
outcome.players,
outcome.error,
outcome.state,
outcome.restartedNow ? 1 : 0,
now,
row.deployment_id,
row.server
@@ -124,7 +119,7 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
});
}
// Finish deployments where every server is now non-pending
// Finish deployments where every server row is non-pending/non-failed.
db()
.prepare(
`UPDATE deployments
@@ -138,7 +133,6 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
)
.run(nowIso());
// Prune rows older than DEPLOY_KEEP_DAYS that have no pending/failed servers
const cutoff = new Date(Date.now() - config.deployKeepDays * 24 * 60 * 60 * 1000).toISOString();
db()
.prepare(
@@ -153,5 +147,41 @@ export async function sweep(): Promise<{ checked: number; restarted: number; hel
.run(cutoff);
emit({ type: 'sweep-tick', frozen: false });
return { checked, restarted, held, failed };
return { checked, restarted, healthy, held, failed };
}
// Phase 1: wait for the server to be empty, then Komodo-restart it.
async function runRestartPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
const players = await rconGetPlayerCount(server);
if (players === null) {
return { state: 'pending', players: null, error: 'rcon unreachable', restartedNow: false };
}
if (players > 0) {
return { state: 'pending', players, error: null, restartedNow: false };
}
const r = await komodoRestart(server);
if (r.ok) {
// Restart triggered — flip restarted_at so the next tick runs the
// health phase. Still 'pending' overall until health confirms.
return { state: 'pending', players: 0, error: null, restartedNow: true };
}
if (r.kind === 'no-name') {
return { state: 'failed', players: 0, error: r.message, restartedNow: false };
}
return { state: 'failed', players: 0, error: `komodo: ${r.message}`, restartedNow: false };
}
// Phase 2: after Komodo-restart, wait until the container reports running AND
// RCON is answering. Only then does the deployment_server row transition to
// 'done'. If either check fails we stay pending and try again next tick.
async function runHealthPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
const h = await komodoInspectHealthy(server);
if (!h.ok) {
return { state: 'pending', players: null, error: `unhealthy: ${h.reason}`, restartedNow: false };
}
const rcon = await rconStatusResult(server);
if (!rcon.ok) {
return { state: 'pending', players: null, error: `rcon not answering: ${rcon.reason}`, restartedNow: false };
}
return { state: 'done', players: rcon.status.players, error: null, restartedNow: false };
}
+32
View File
@@ -0,0 +1,32 @@
import { json } from '@sveltejs/kit';
import { runDriftDeployment, scanFleetDrift } from '$lib/server/fleet/drift.js';
import { kick } from '$lib/server/pipeline/worker.js';
// GET returns the current drift plan for the whole fleet without touching disk.
export async function GET() {
const entries = await scanFleetDrift();
const byServer = new Map<string, Array<{ relPath: string; size: number; sha256: string }>>();
for (const e of entries) {
const arr = byServer.get(e.server) ?? [];
arr.push({ relPath: e.relPath, size: e.size, sha256: e.sha256 });
byServer.set(e.server, arr);
}
return json({
totalDrift: entries.length,
servers: [...byServer.entries()].map(([server, files]) => ({ server, files }))
});
}
// POST removes drift and creates a drift deployment. Body: { servers?: string[], dryRun?: boolean }
export async function POST({ request }: { request: Request }) {
const body = (await request
.json()
.catch(() => ({}))) as { servers?: string[]; dryRun?: boolean };
const result = await runDriftDeployment({
servers: body.servers,
dryRun: body.dryRun ?? false,
triggeredBy: 'ui'
});
if (result.deploymentId) void kick();
return json({ ok: true, result });
}
+124 -1
View File
@@ -17,13 +17,59 @@
servers: ServerRow[];
}
interface DriftFile { relPath: string; size: number; sha256: string }
interface DriftServer { server: string; files: DriftFile[] }
interface DriftPreview { totalDrift: number; servers: DriftServer[] }
let deployments = $state<Deployment[]>([]);
let loading = $state(true);
let drift = $state<DriftPreview | null>(null);
let scanning = $state(false);
let running = $state(false);
let flash = $state<string | null>(null);
onMount(async () => {
async function loadDeployments(): Promise<void> {
const res = await fetch('/api/deployments');
const body = (await res.json()) as { deployments: Deployment[] };
deployments = body.deployments;
}
async function scanDrift(): Promise<void> {
scanning = true;
try {
const res = await fetch('/api/drift');
drift = (await res.json()) as DriftPreview;
} finally {
scanning = false;
}
}
async function runDrift(): Promise<void> {
if (!drift || drift.totalDrift === 0) return;
if (!confirm(`Remove ${drift.totalDrift} untracked file(s) across ${drift.servers.length} server(s)?`)) return;
running = true;
flash = null;
try {
const res = await fetch('/api/drift', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({})
});
const body = (await res.json()) as {
ok: boolean;
error?: string;
result?: { deploymentId: string | null; totalRemoved: number; totalServers: number };
};
if (!body.ok) { flash = body.error ?? 'drift run failed'; return; }
flash = `removed ${body.result?.totalRemoved ?? 0} files across ${body.result?.totalServers ?? 0} servers`;
await Promise.all([loadDeployments(), scanDrift()]);
} finally {
running = false;
}
}
onMount(async () => {
await Promise.all([loadDeployments(), scanDrift()]);
loading = false;
});
@@ -52,6 +98,35 @@
{#if loading}
<p class="empty">loading…</p>
{:else}
<section class="drift-panel">
<div class="drift-head">
<h2>Drift{drift ? ` · ${drift.totalDrift} untracked file${drift.totalDrift === 1 ? '' : 's'}` : ''}</h2>
<div class="drift-actions">
<button onclick={scanDrift} disabled={scanning || running}>{scanning ? 'scanning…' : 'rescan'}</button>
<button class="primary" onclick={runDrift} disabled={running || scanning || !drift || drift.totalDrift === 0}>
{running ? 'removing…' : 'remove drift'}
</button>
</div>
</div>
{#if drift === null}
<p class="empty">scan hasn't run yet</p>
{:else if drift.totalDrift === 0}
<p class="empty">inventory matches disk — no drift</p>
{:else}
{#each drift.servers as s (s.server)}
<div class="drift-server">
<div class="drift-server-head mono">{s.server} · {s.files.length} file{s.files.length === 1 ? '' : 's'}</div>
<div class="drift-files">
{#each s.files as f (f.relPath)}
<span class="drift-file mono" title="{f.size}B · {f.sha256.slice(0, 12)}">{f.relPath}</span>
{/each}
</div>
</div>
{/each}
{/if}
{#if flash}<p class="flash">{flash}</p>{/if}
</section>
<section>
<h2>Active</h2>
{#if active.length === 0}
@@ -168,4 +243,52 @@
font-family: var(--font-mono);
font-size: var(--text-base);
}
.drift-panel {
background: var(--bg-nav);
border: 1px solid var(--border);
border-radius: var(--r-md);
padding: var(--pad-panel);
margin-bottom: var(--gap-section);
}
.drift-head { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 14px; }
.drift-head h2 { margin: 0; }
.drift-actions { display: flex; gap: 8px; }
.drift-actions button {
background: var(--bg-card);
color: var(--ink-100);
border: 1px solid var(--border);
border-radius: var(--r-sm);
padding: 8px 16px;
font-family: var(--font-body);
font-size: var(--text-sm);
cursor: pointer;
}
.drift-actions button:hover:not(:disabled) { border-color: var(--accent); box-shadow: var(--accent-glow); }
.drift-actions button.primary {
background: linear-gradient(180deg, var(--danger-500), #a01e39);
border-color: rgba(255, 48, 80, 0.6);
color: var(--cream);
font-weight: 600;
}
.drift-actions button:disabled { opacity: 0.4; cursor: default; }
.drift-server {
background: var(--bg-card);
border: 1px solid var(--border);
border-radius: var(--r-sm);
padding: 10px 14px;
margin: 8px 0;
}
.drift-server-head { color: var(--cream); font-size: var(--text-mono); margin-bottom: 6px; }
.drift-files { display: flex; flex-wrap: wrap; gap: 6px; }
.drift-file {
background: rgba(255, 48, 80, 0.08);
border: 1px solid rgba(255, 48, 80, 0.25);
color: var(--danger-500);
padding: 3px 8px;
border-radius: var(--r-sm);
font-size: var(--text-mono-sm);
}
.flash { color: var(--spark-400); font-family: var(--font-mono); font-size: var(--text-mono); margin-top: 12px; }
.mono { font-family: var(--font-mono); }
</style>
+1 -1
View File
@@ -53,6 +53,6 @@ describe('sweep', () => {
await seedServerRegistry();
const result = await sweep();
expect(result).toEqual({ checked: 0, restarted: 0, held: 0, failed: 0 });
expect(result).toEqual({ checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 });
});
});