import { getVersionContent } from './scripts.js'; import { readdir, readFile, stat, nativeFileManifest, type FileFingerprint } from '../backends/files.js'; import { randomUUID } from 'node:crypto'; import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js'; import { join, posix, relative, sep } from 'node:path'; import { configFileOf, destOfKind, serverDir, type ArtifactKind } from '../config.js'; import { db } from '../db/index.js'; import { emit } from '../pipeline/bus.js'; import { sha256Buffer } from './files.js'; import { getServerScript } from './inventory.js'; import { getBaselineForScope, getVersion } from './scripts.js'; import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js'; import { selectedZombieMap } from './map-aliases.js'; import { getServer, listServers } from './servers.js'; import { promotedArtifactMetadata } from './promoted-artifacts.js'; import { queueFileChange } from './staged-files.js'; import { trackActivity } from '../pipeline/activity.js'; /** Queue a small batch for the agent's next poll, preserving input order. */ async function scanBatch(items: readonly T[], work: (item: T) => Promise): Promise { const results: R[] = new Array(items.length); let cursor = 0; let failed = false; async function pump() { while (!failed && cursor < items.length) { const index = cursor++; try { results[index] = await work(items[index]); } catch (error) { failed = true; throw error; } } } const workers = await Promise.allSettled(Array.from({ length: Math.min(4, items.length) }, pump)); const failure = workers.find(result => result.status === 'rejected'); if (failure?.status === 'rejected') throw failure.reason; return results; } // Destinations drift scans, in a stable order so the UI shows scripts first. // Baseline shares the config destination — one physical dir, two logical kinds. const DRIFT_DESTS: ReadonlyArray = ['script', 'config', 'plugin']; // Filenames we never treat as drift. dedicated.cfg is placed by Plutonium for // multiplayer and regenerates if deleted; we manage dedicated_sp.cfg (zombies) // via config-compose instead. const DRIFT_IGNORE_FILENAMES = new Set(['dedicated.cfg']); // Three flavors of drift: // 'untracked' — file outside inventory and promoted scope (delete it) // 'replace' — managed target exists but needs installation/tracking // 'missing' — artifact promoted for this server's scope, but file is gone // (write it back from the DB) export type DriftKind = 'untracked' | 'missing' | 'replace'; export interface DriftEntry { server: string; relPath: string; kind: ArtifactKind; driftKind: DriftKind; // Populated for on-disk detections (untracked) and missing configs where // we compute the expected content. size: number; sha256: string; // Populated for missing/replacement entries so runDriftDeployment can write the // right content back. scriptId?: number; scriptVersionId?: number | null; } function toPosix(p: string): string { return p.split(sep).join(posix.sep); } function hasActiveBeta(server: string): boolean { return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server); } async function requireServerDirectory(server: string): Promise { try { if (!(await stat(serverDir(server))).isDirectory()) throw new Error('path is not a directory'); } catch (error) { throw new Error(`Server files unavailable for ${server}: ${(error as Error).message}`); } } async function readExistingFile(path: string): Promise { return readFile(path).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; }); } async function fingerprint(path: string, manifest: Map | null): Promise { if (manifest) return manifest.get(path) ?? null; const content = await readExistingFile(path); return content === null ? null : { size: content.byteLength, sha256: sha256Buffer(content) }; } async function walkFiles(root: string): Promise { const out: string[] = []; let directories = [root]; while (directories.length) { const children = await scanBatch(directories, async dir => { let entries; try { entries = await readdir(dir, { withFileTypes: true }); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; throw error; } const next: string[] = []; for (const entry of entries) { if (DRIFT_IGNORE_FILENAMES.has(entry.name)) continue; const abs = join(dir, entry.name); if (entry.isDirectory()) next.push(abs); else if (entry.isFile()) out.push(toPosix(relative(root, abs))); } return next; }); directories = children.flat(); } return out.sort(); } /** * Scan every managed destination tree on a server (scripts, configs, plugins) * and compare disk contents with promoted artifacts and inventory. An active * beta owns its server's files until promotion or rollback; defer the whole * server so reconciliation cannot replace experimental config or missing beta * artifacts with the promoted versions. */ export async function scanServerDrift(server: string): Promise { return trackActivity(`drift:${server}`, `Checking files on ${server}`, () => scanServerFiles(server)); } async function scanServerFiles(server: string): Promise { const srv = getServer(server); if (!srv || srv.destroyed_at || hasActiveBeta(server)) return []; // A missing mount is unavailable evidence, not an empty managed tree. const expectedPaths = promotedArtifactMetadata(srv.game, srv.map).map(({ kind, script }) => join(destOfKind(server, kind), script.target_rel_path)); if (srv.game && getBaselineForScope('game', srv.game)?.current_version_id) expectedPaths.push(join(destOfKind(server, 'config'), configFileOf(server))); if (srv.created_by && srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) expectedPaths.push(join(destOfKind(server, 'config'), MAP_CONFIG_FILE)); const manifest = await nativeFileManifest(serverDir(server), DRIFT_DESTS.map(kind => destOfKind(server, kind)), [...DRIFT_IGNORE_FILENAMES], expectedPaths); if (!manifest) await requireServerDirectory(server); const drift = await scanManagedForServer(server, manifest); // Previous inventory is an audit of installed bytes, not permission to // retain artifacts whose scope was moved or removed from the library. const managed = new Set(promotedArtifactMetadata(srv.game, srv.map) .map(({ kind, script }) => `${kind}:${script.target_rel_path}`)); const baseline = srv.game ? getBaselineForScope('game', srv.game) : undefined; if (baseline?.current_version_id) managed.add(`config:${configFileOf(server)}`); // Without a promoted baseline, Plutainer owns the seeded config. Keep it // until a baseline is promoted; startup arguments handle map selection. const keepDefaultConfig = srv?.game && selectedZombieMap(srv.game, srv.map, srv.map_slug) && !baseline?.current_version_id; // Scan for untracked files only after resolving managed targets. for (const kind of DRIFT_DESTS) { const dest = destOfKind(server, kind); const onDisk = manifest ? [...manifest.keys()].filter(path => path.startsWith(`${dest}${sep}`)).map(path => toPosix(relative(dest, path))) .filter(path => !path.split('/').some(name => DRIFT_IGNORE_FILENAMES.has(name))).sort() : await walkFiles(dest); if (onDisk.length === 0) continue; const untracked = await scanBatch(onDisk, async (relPath): Promise => { if (managed.has(`${kind}:${relPath}`)) return null; if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server)) return null; if (kind === 'config' && relPath === MAP_CONFIG_FILE && srv?.created_by && srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) return null; const abs = join(dest, relPath); try { const info = await fingerprint(abs, manifest); if (!info) return null; return { server, relPath, ...info, kind, driftKind: 'untracked' }; } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; // A file disappeared between walk and stat; the next scan sees it. } return null; }); drift.push(...untracked.filter((entry): entry is DriftEntry => entry !== null)); } if (!manifest) await requireServerDirectory(server); return hasActiveBeta(server) ? [] : drift; } // Every promoted artifact whose scope matches this server's game/map should // have its target file present on disk. Anything absent is 'missing' drift. // Configs are handled as a composed layer (baseline + overlays → one cfg). async function scanManagedForServer(server: string, manifest: Map | null): Promise { const srv = getServer(server); if (!srv) return []; const out: DriftEntry[] = []; // Provisioned servers load this generated file after their main cfg. It // remains managed even when there is no promoted baseline yet. const mapConfig = srv.created_by && srv.game_mode === 'zombie' ? getServerMapConfig(srv.game, srv.map, srv.map_slug) : null; if (mapConfig) { const abs = join(destOfKind(server, 'config'), MAP_CONFIG_FILE); const content = await fingerprint(abs, manifest); if (content?.sha256 !== sha256Buffer(mapConfig)) { out.push({ server, relPath: MAP_CONFIG_FILE, kind: 'config', driftKind: content ? 'replace' : 'missing', size: mapConfig.byteLength, sha256: sha256Buffer(mapConfig) }); } } // Inventory records the last managed write, not the current disk bytes. // Check both so edits and outdated versions are detected on tracked files. const managed = await scanBatch(promotedArtifactMetadata(srv.game, srv.map), async ({ kind, script, version }): Promise => { const content = await fingerprint(join(destOfKind(server, kind), script.target_rel_path), manifest); const inventory = getServerScript(server, script.target_rel_path); if (content?.sha256 === version.sha256 && inventory?.sha256 === version.sha256 && inventory.script_version_id === version.id) return null; return { server, relPath: script.target_rel_path, kind, driftKind: content ? 'replace' : 'missing', size: version.size, sha256: version.sha256, scriptId: script.id, scriptVersionId: version.id }; }); out.push(...managed.filter((entry): entry is DriftEntry => entry !== null)); // Composed cfg: the effective dedicated_sp.cfg is baseline + all promoted // overlays for this server's scope. It's "missing" when either the file // isn't on disk at all, or its sha256 differs from what we'd compose. // Only meaningful when there's a promoted baseline for this server's game. if (srv.game) { const baseline = getBaselineForScope('game', srv.game); if (baseline?.current_version_id) { const composed = composeExpectedCfg(srv.game, srv.map, srv.map_slug); if (composed) { const dest = destOfKind(server, 'config'); const targetRel = configFileOf(server); const abs = join(dest, targetRel); const inv = getServerScript(server, targetRel); const content = await fingerprint(abs, manifest); const onDiskMatches = content?.sha256 === composed.sha256 && inv?.sha256 === composed.sha256; if (!onDiskMatches) { out.push({ server, relPath: targetRel, kind: 'config', driftKind: content ? 'replace' : 'missing', size: composed.content.byteLength, sha256: composed.sha256, scriptId: baseline.id, scriptVersionId: baseline.current_version_id }); } } } } return out; } // Recompute what dedicated_sp.cfg SHOULD look like on this server given the // current promoted baseline + overlays. Returns null if no baseline exists. // Resolve the content we should install for a managed drift entry. For // scripts/plugins, that's the current version's BLOB. For configs, recompose // the effective cfg from baseline + overlays. async function resolveManagedEntry(server: string, entry: DriftEntry): Promise { if (entry.kind === 'config') { const srv = getServer(server); if (!srv?.game) return null; if (entry.relPath === MAP_CONFIG_FILE) return getServerMapConfig(srv.game, srv.map, srv.map_slug); const composed = composeExpectedCfg(srv.game, srv.map, srv.map_slug); return composed?.content ?? null; } if (!entry.scriptVersionId) return null; const ver = getVersion(entry.scriptVersionId); return ver ? getVersionContent(ver.id) : null; } function composeExpectedCfg( game: string | null, map: string | null, mapSlug?: string | null ): { content: Buffer; sha256: string } | null { const content = getPromotedServerConfig(game, map, mapSlug); if (!content) return null; return { content, sha256: sha256Buffer(content) }; } /** * Scan the whole fleet. Servers are read from the DB (registered fleet), not * the on-disk SERVERS_ROOT listing, so unregistered folders don't leak in. */ export async function scanFleetDrift(): Promise { const servers = listServers().map((s) => s.name); const out: DriftEntry[] = []; for (const name of servers) { const rows = await scanServerDrift(name); out.push(...rows); } return out; } export interface RunDriftInput { servers?: string[]; // if omitted, scans the whole fleet dryRun?: boolean; triggeredBy?: 'ui' | 'sweep' | 'cron'; expectedFingerprint?: string; } export class DriftPlanChanged extends Error { constructor(public plan: DriftPlan) { super('The files or target servers changed. Nothing was changed. Review the updated plan before continuing.'); } } function driftTargets(servers?: string[]): string[] { const registered = listServers().map(server => server.name); if (servers && (!servers.length || servers.some(name => !registered.includes(name)))) throw new Error('Select at least one registered server. Refresh the fleet if a server was removed.'); return [...new Set(servers ?? registered)].sort(); } async function describePlan(servers: string[], entries: DriftEntry[]): Promise { const files: DriftPlanFile[] = []; for (const entry of entries) { const before = await readFile(join(destOfKind(entry.server, entry.kind), entry.relPath)).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; }); const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(entry.server, entry); files.push({ server: entry.server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind, beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null, beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null }); } return buildPlan(servers, files); } function buildPlan(servers: string[], files: DriftPlanFile[]): DriftPlan { files.sort((a, b) => `${a.server}:${a.kind}:${a.relPath}`.localeCompare(`${b.server}:${b.kind}:${b.relPath}`)); return { servers, files, scannedAt: new Date().toISOString(), fingerprint: sha256Buffer(Buffer.from(JSON.stringify({ servers, files }))) }; } export async function previewDrift(servers?: string[]): Promise { const targets = driftTargets(servers); const entries: DriftEntry[] = []; for (const server of targets) entries.push(...await scanServerDrift(server)); return describePlan(targets, entries); } export interface RunDriftResult { deploymentId: string | null; // `removed` and `restored` describe queued changes. Provisioning applies // them only after the server passes its activation gates. perServer: Array<{ server: string; removed: string[]; restored: string[]; skipped: string[] }>; totalRemoved: number; totalRestored: number; totalServers: number; dryRun: boolean; } function deploymentIdNow(): string { const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z'); return `dep-${stamp}-drift-${randomUUID().slice(0, 8)}`; } export interface PreparedDriftCampaign { plan: DriftPlan; files: Array<{ file: DriftPlanFile; before: Buffer | null; after: Buffer | null }>; } /** Capture a rebuild's file evidence without changing the running instance. */ export async function prepareDriftCampaign(server: string): Promise { const servers = driftTargets([server]); const entries = await scanServerDrift(server); const files = await scanBatch(entries, async entry => { const before = await readExistingFile(join(destOfKind(server, entry.kind), entry.relPath)); const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(server, entry); if (entry.driftKind !== 'untracked' && !after) throw new Error(`Cannot resolve promoted content for ${entry.relPath}`); return { before, after, file: { server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind, beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null, beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null } }; }); if (entries.length) await requireServerDirectory(server); if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred'); return { plan: buildPlan(servers, files.map(({ file }) => file)), files }; } /** Record the audited plan atomically with a queued rebuild, never a second restart. */ export function createDriftRebuildCampaign( server: string, operationId: string, incidentId: string | number, prepared: PreparedDriftCampaign ): string | null { if (!prepared.files.length) return null; if (prepared.plan.servers.length !== 1 || prepared.plan.servers[0] !== server || prepared.files.some(({ file }) => file.server !== server)) throw new Error('Drift evidence belongs to another server'); if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred'); return db().transaction(() => { if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server = ? AND action = 'rebuild'").get(operationId, server)) { throw new Error('Drift campaign requires the matching rebuild operation'); } const id = deploymentIdNow(); db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')") .run(id, new Date().toISOString()); const insertFile = db().prepare(`INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`); for (const { file, before, after } of prepared.files) { insertFile.run(id, server, after === null ? 'removed' : before === null ? 'added' : 'updated', file.relPath, file.beforeSha256, file.afterSha256, before, after); } db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id) VALUES (?, ?, ?, 'pending', ?)`).run(id, server, prepared.files.length, operationId); db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)") .run(operationId, id, server); db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)') .run(id, `Watchdog ${incidentId}: captured drift evidence; rebuild ${operationId} will install the latest promoted setup. Files remain unchanged until provisioning rebuilds the instance.`); emit({ type: 'deployment-opened', deploymentId: id, action: 'drift', servers: [server] }); return id; })(); } /** * Detect drift on the requested servers (or the whole fleet) and either * report the plan (dryRun) or stage the changes as a single deployment. * Provisioning applies them and restarts each affected server once it's empty. */ export async function runDriftDeployment(input: RunDriftInput = {}): Promise { const { dryRun = false, triggeredBy = 'ui' } = input; const scanTargets = driftTargets(input.servers); const perServer: Array<{ server: string; removed: string[]; restored: string[]; skipped: string[]; entries: DriftEntry[]; }> = []; for (const name of scanTargets) { const entries = await scanServerDrift(name); perServer.push({ server: name, removed: [], restored: [], skipped: [], entries }); } // Recheck identities, existing bytes, and replacement bytes before any write. const plan = input.expectedFingerprint ? await describePlan(scanTargets, perServer.flatMap(server => server.entries)) : null; if (plan && plan.fingerprint !== input.expectedFingerprint) throw new DriftPlanChanged(plan); const totalDrift = perServer.reduce((sum, s) => sum + s.entries.length, 0); if (totalDrift === 0 || dryRun) { return { deploymentId: null, perServer: perServer.map((s) => ({ server: s.server, removed: s.entries.filter((e) => e.driftKind === 'untracked').map((e) => e.relPath), restored: s.entries.filter((e) => e.driftKind !== 'untracked').map((e) => e.relPath), skipped: [] })), totalRemoved: perServer.reduce( (n, s) => n + s.entries.filter((e) => e.driftKind === 'untracked').length, 0 ), totalRestored: perServer.reduce( (n, s) => n + s.entries.filter((e) => e.driftKind !== 'untracked').length, 0 ), totalServers: perServer.filter((s) => s.entries.length > 0).length, dryRun }; } // Commit. Create the deployment row FIRST so audit rows (which FK to it) // can be inserted as we go. const depId = deploymentIdNow(); const now = new Date().toISOString(); db().prepare( `INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, ?)` ).run(depId, now, triggeredBy); const insertServer = db().prepare( `INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error) VALUES (?, ?, ?, 'pending', NULL, NULL, NULL, 0, NULL)` ); const insertFile = db().prepare( `INSERT INTO deployment_files (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` ); const affected: string[] = []; for (const s of perServer) { if (s.entries.length === 0) continue; if (hasActiveBeta(s.server)) { s.skipped.push('Active beta owns this server; reconciliation deferred'); continue; } for (const entry of s.entries) { const dest = destOfKind(s.server, entry.kind); const abs = join(dest, entry.relPath); try { await requireServerDirectory(s.server); if (hasActiveBeta(s.server)) { s.skipped.push(`${entry.relPath}: active beta owns this server; reconciliation deferred`); continue; } const reviewed = plan?.files.find(file => file.server === s.server && file.kind === entry.kind && file.relPath === entry.relPath); if (reviewed) { const current = await readFile(abs).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; }); if ((current ? sha256Buffer(current) : null) !== reviewed.beforeSha256) { s.skipped.push(`${entry.relPath}: changed after review; scan and review again`); continue; } } if (entry.driftKind === 'untracked') { // Capture the reviewed deletion without touching the live tree. const content = await readFile(abs); if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred'); db().transaction(() => { const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, content, null); queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server, kind: entry.kind, relPath: entry.relPath, content: null, inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid), expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content) }); })(); s.removed.push(entry.relPath); } else { // Persist exact replacement bytes so waiting cannot change the plan. const written = await resolveManagedEntry(s.server, entry); if (!written) { s.skipped.push(`${entry.relPath}: missing artifact could not be resolved from the DB`); continue; } if (reviewed && sha256Buffer(written) !== reviewed.afterSha256) { s.skipped.push(`${entry.relPath}: promoted content changed after review; scan and review again`); continue; } const before = await readFile(abs).catch((err: NodeJS.ErrnoException) => { if (err.code === 'ENOENT') return null; throw err; }); if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred'); db().transaction(() => { const file = insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath, before ? sha256Buffer(before) : null, sha256Buffer(written), before, written); queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server, kind: entry.kind, relPath: entry.relPath, content: written, scriptVersionId: entry.kind === 'config' ? null : entry.scriptVersionId ?? null, inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid), expectedBeforeSha: reviewed ? reviewed.beforeSha256 : before ? sha256Buffer(before) : null }); })(); s.restored.push(entry.relPath); } } catch (err) { s.skipped.push(`${entry.relPath}: ${(err as Error).message}`); } } const changes = s.removed.length + s.restored.length; if (changes > 0) { insertServer.run(depId, s.server, changes); affected.push(s.server); } } if (affected.length === 0) { // Everything failed. Roll back the empty deployment row so it doesn't // dangle in the UI as a mysterious no-op. db().prepare('DELETE FROM deployments WHERE id = ?').run(depId); return { deploymentId: null, perServer: perServer.map((s) => ({ server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped })), totalRemoved: 0, totalRestored: 0, totalServers: 0, dryRun: false }; } emit({ type: 'deployment-opened', deploymentId: depId, action: 'drift', servers: affected }); return { deploymentId: depId, perServer: perServer.map((s) => ({ server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped })), totalRemoved: perServer.reduce((sum, s) => sum + s.removed.length, 0), totalRestored: perServer.reduce((sum, s) => sum + s.restored.length, 0), totalServers: affected.length, dryRun: false }; }