Files
electric-cherry/src/lib/server/fleet/drift.ts
T
josh e5c0fda1ed Add native host management and role-based operator authentication
Introduce an outbound Python host agent, Debian installer, durable remote
operations, immutable game-file libraries, host-aware deployments and ports,
and occupancy-safe campaigns and draining. Native hosts do not require Komodo.

Add administrator, operator and viewer accounts, protected management routes,
secure sessions, password recovery, audit history and separate telemetry tokens.
Document first-admin setup, reporter migration and live-host acceptance checks.

Validation: 1,343 TypeScript tests and 39 agent tests passed, along with type
checking, the production build, compiled SSR/API smoke tests and administrator
recovery. Real Debian/Docker T5/T6 acceptance remains unrun.
2026-10-03 11:42:36 -04:00

553 lines
26 KiB
TypeScript

import { readdir, readFile, stat } from '../backends/files.js';
import { randomUUID } from 'node:crypto';
import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js';
import { join, posix, relative, sep } from 'node:path';
import { configFileOf, destOfKind, serverDir, type ArtifactKind } from '../config.js';
import { db } from '../db/index.js';
import { emit } from '../pipeline/bus.js';
import { sha256Buffer } from './files.js';
import { getServerScript } from './inventory.js';
import { getBaselineForScope, getVersion } from './scripts.js';
import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
import { selectedZombieMap } from './map-aliases.js';
import { getServer, listServers } from './servers.js';
import { promotedArtifacts } from './promoted-artifacts.js';
import { queueFileChange } from './staged-files.js';
// Destinations drift scans, in a stable order so the UI shows scripts first.
// Baseline shares the config destination — one physical dir, two logical kinds.
const DRIFT_DESTS: ReadonlyArray<ArtifactKind> = ['script', 'config', 'plugin'];
// Filenames we never treat as drift. dedicated.cfg is placed by Plutonium for
// multiplayer and regenerates if deleted; we manage dedicated_sp.cfg (zombies)
// via config-compose instead.
const DRIFT_IGNORE_FILENAMES = new Set(['dedicated.cfg']);
// Three flavors of drift:
// 'untracked' — file outside inventory and promoted scope (delete it)
// 'replace' — managed target exists but needs installation/tracking
// 'missing' — artifact promoted for this server's scope, but file is gone
// (write it back from the DB)
export type DriftKind = 'untracked' | 'missing' | 'replace';
export interface DriftEntry {
server: string;
relPath: string;
kind: ArtifactKind;
driftKind: DriftKind;
// Populated for on-disk detections (untracked) and missing configs where
// we compute the expected content.
size: number;
sha256: string;
// Populated for missing/replacement entries so runDriftDeployment can write the
// right content back.
scriptId?: number;
scriptVersionId?: number | null;
}
function toPosix(p: string): string {
return p.split(sep).join(posix.sep);
}
function hasActiveBeta(server: string): boolean {
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server);
}
async function requireServerDirectory(server: string): Promise<void> {
try {
if (!(await stat(serverDir(server))).isDirectory()) throw new Error('path is not a directory');
} catch (error) {
throw new Error(`Server files unavailable for ${server}: ${(error as Error).message}`);
}
}
async function readExistingFile(path: string): Promise<Buffer | null> {
return readFile(path).catch((error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return null;
throw error;
});
}
async function walkFiles(root: string): Promise<string[]> {
const out: string[] = [];
async function recur(dir: string): Promise<void> {
let entries;
try {
entries = await readdir(dir, { withFileTypes: true });
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return;
throw error;
}
for (const e of entries) {
if (DRIFT_IGNORE_FILENAMES.has(e.name)) continue;
const abs = join(dir, e.name);
if (e.isDirectory()) await recur(abs);
else if (e.isFile()) out.push(toPosix(relative(root, abs)));
}
}
await recur(root);
return out.sort();
}
/**
* Scan every managed destination tree on a server (scripts, configs, plugins)
* and compare disk contents with promoted artifacts and inventory. An active
* beta owns its server's files until promotion or rollback; defer the whole
* server so reconciliation cannot replace experimental config or missing beta
* artifacts with the promoted versions.
*/
export async function scanServerDrift(server: string): Promise<DriftEntry[]> {
const srv = getServer(server);
if (!srv || srv.destroyed_at || hasActiveBeta(server)) return [];
// A missing mount is unavailable evidence, not an empty managed tree.
await requireServerDirectory(server);
const drift = await scanManagedForServer(server);
// Previous inventory is an audit of installed bytes, not permission to
// retain artifacts whose scope was moved or removed from the library.
const managed = new Set(promotedArtifacts(srv.game, srv.map)
.map(({ kind, script }) => `${kind}:${script.target_rel_path}`));
const baseline = srv.game ? getBaselineForScope('game', srv.game) : undefined;
if (baseline?.current_version_id) managed.add(`config:${configFileOf(server)}`);
// Without a promoted baseline, Plutainer owns the seeded config. Keep it
// until a baseline is promoted; startup arguments handle map selection.
const keepDefaultConfig = srv?.game
&& selectedZombieMap(srv.game, srv.map, srv.map_slug)
&& !baseline?.current_version_id;
// Scan for untracked files only after resolving managed targets.
for (const kind of DRIFT_DESTS) {
const dest = destOfKind(server, kind);
const onDisk = await walkFiles(dest);
if (onDisk.length === 0) continue;
for (const relPath of onDisk) {
if (managed.has(`${kind}:${relPath}`)) continue;
if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server)) continue;
if (kind === 'config' && relPath === MAP_CONFIG_FILE && srv?.created_by
&& srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) continue;
const abs = join(dest, relPath);
try {
const st = await stat(abs);
const buf = await readFile(abs);
drift.push({
server, relPath,
size: st.size,
sha256: sha256Buffer(buf),
kind,
driftKind: 'untracked'
});
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
// A file disappeared between walk and stat; the next scan sees it.
}
}
}
await requireServerDirectory(server);
return hasActiveBeta(server) ? [] : drift;
}
// Every promoted artifact whose scope matches this server's game/map should
// have its target file present on disk. Anything absent is 'missing' drift.
// Configs are handled as a composed layer (baseline + overlays → one cfg).
async function scanManagedForServer(server: string): Promise<DriftEntry[]> {
const srv = getServer(server);
if (!srv) return [];
const out: DriftEntry[] = [];
// Provisioned servers load this generated file after their main cfg. It
// remains managed even when there is no promoted baseline yet.
const mapConfig = srv.created_by && srv.game_mode === 'zombie'
? getServerMapConfig(srv.game, srv.map, srv.map_slug) : null;
if (mapConfig) {
const abs = join(destOfKind(server, 'config'), MAP_CONFIG_FILE);
const content = await readExistingFile(abs);
if (!content?.equals(mapConfig)) {
out.push({ server, relPath: MAP_CONFIG_FILE, kind: 'config', driftKind: content ? 'replace' : 'missing',
size: mapConfig.byteLength, sha256: sha256Buffer(mapConfig) });
}
}
// Inventory records the last managed write, not the current disk bytes.
// Check both so edits and outdated versions are detected on tracked files.
for (const { kind, script, version } of promotedArtifacts(srv.game, srv.map)) {
const content = await readExistingFile(join(destOfKind(server, kind), script.target_rel_path));
const inventory = getServerScript(server, script.target_rel_path);
if (content?.equals(version.content) && inventory?.sha256 === version.sha256
&& inventory.script_version_id === version.id) continue;
out.push({
server, relPath: script.target_rel_path, kind,
driftKind: content ? 'replace' : 'missing',
size: version.size, sha256: version.sha256,
scriptId: script.id, scriptVersionId: version.id
});
}
// Composed cfg: the effective dedicated_sp.cfg is baseline + all promoted
// overlays for this server's scope. It's "missing" when either the file
// isn't on disk at all, or its sha256 differs from what we'd compose.
// Only meaningful when there's a promoted baseline for this server's game.
if (srv.game) {
const baseline = getBaselineForScope('game', srv.game);
if (baseline?.current_version_id) {
const composed = composeExpectedCfg(srv.game, srv.map, srv.map_slug);
if (composed) {
const dest = destOfKind(server, 'config');
const targetRel = configFileOf(server);
const abs = join(dest, targetRel);
const inv = getServerScript(server, targetRel);
const content = await readExistingFile(abs);
const onDiskMatches = content?.equals(composed.content) && inv?.sha256 === composed.sha256;
if (!onDiskMatches) {
out.push({
server,
relPath: targetRel,
kind: 'config',
driftKind: content ? 'replace' : 'missing',
size: composed.content.byteLength,
sha256: composed.sha256,
scriptId: baseline.id,
scriptVersionId: baseline.current_version_id
});
}
}
}
}
return out;
}
// Recompute what dedicated_sp.cfg SHOULD look like on this server given the
// current promoted baseline + overlays. Returns null if no baseline exists.
// Resolve the content we should install for a managed drift entry. For
// scripts/plugins, that's the current version's BLOB. For configs, recompose
// the effective cfg from baseline + overlays.
async function resolveManagedEntry(server: string, entry: DriftEntry): Promise<Buffer | null> {
if (entry.kind === 'config') {
const srv = getServer(server);
if (!srv?.game) return null;
if (entry.relPath === MAP_CONFIG_FILE) return getServerMapConfig(srv.game, srv.map, srv.map_slug);
const composed = composeExpectedCfg(srv.game, srv.map, srv.map_slug);
return composed?.content ?? null;
}
if (!entry.scriptVersionId) return null;
const ver = getVersion(entry.scriptVersionId);
return ver?.content ?? null;
}
function composeExpectedCfg(
game: string | null,
map: string | null,
mapSlug?: string | null
): { content: Buffer; sha256: string } | null {
const content = getPromotedServerConfig(game, map, mapSlug);
if (!content) return null;
return { content, sha256: sha256Buffer(content) };
}
/**
* Scan the whole fleet. Servers are read from the DB (registered fleet), not
* the on-disk SERVERS_ROOT listing, so unregistered folders don't leak in.
*/
export async function scanFleetDrift(): Promise<DriftEntry[]> {
const servers = listServers().map((s) => s.name);
const out: DriftEntry[] = [];
for (const name of servers) {
const rows = await scanServerDrift(name);
out.push(...rows);
}
return out;
}
export interface RunDriftInput {
servers?: string[]; // if omitted, scans the whole fleet
dryRun?: boolean;
triggeredBy?: 'ui' | 'sweep' | 'cron';
expectedFingerprint?: string;
}
export class DriftPlanChanged extends Error {
constructor(public plan: DriftPlan) { super('The files or target servers changed. Nothing was changed. Review the updated plan before continuing.'); }
}
function driftTargets(servers?: string[]): string[] {
const registered = listServers().map(server => server.name);
if (servers && (!servers.length || servers.some(name => !registered.includes(name)))) throw new Error('Select at least one registered server. Refresh the fleet if a server was removed.');
return [...new Set(servers ?? registered)].sort();
}
async function describePlan(servers: string[], entries: DriftEntry[]): Promise<DriftPlan> {
const files: DriftPlanFile[] = [];
for (const entry of entries) {
const before = await readFile(join(destOfKind(entry.server, entry.kind), entry.relPath)).catch((error: NodeJS.ErrnoException) => {
if (error.code === 'ENOENT') return null;
throw error;
});
const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(entry.server, entry);
files.push({ server: entry.server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null });
}
return buildPlan(servers, files);
}
function buildPlan(servers: string[], files: DriftPlanFile[]): DriftPlan {
files.sort((a, b) => `${a.server}:${a.kind}:${a.relPath}`.localeCompare(`${b.server}:${b.kind}:${b.relPath}`));
return { servers, files, scannedAt: new Date().toISOString(), fingerprint: sha256Buffer(Buffer.from(JSON.stringify({ servers, files }))) };
}
export async function previewDrift(servers?: string[]): Promise<DriftPlan> {
const targets = driftTargets(servers);
const entries: DriftEntry[] = [];
for (const server of targets) entries.push(...await scanServerDrift(server));
return describePlan(targets, entries);
}
export interface RunDriftResult {
deploymentId: string | null;
// `removed` and `restored` describe queued changes. Provisioning applies
// them only after the server passes its activation gates.
perServer: Array<{ server: string; removed: string[]; restored: string[]; skipped: string[] }>;
totalRemoved: number;
totalRestored: number;
totalServers: number;
dryRun: boolean;
}
function deploymentIdNow(): string {
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
return `dep-${stamp}-drift-${randomUUID().slice(0, 8)}`;
}
export interface PreparedDriftCampaign {
plan: DriftPlan;
files: Array<{ file: DriftPlanFile; before: Buffer | null; after: Buffer | null }>;
}
/** Capture a rebuild's file evidence without changing the running instance. */
export async function prepareDriftCampaign(server: string): Promise<PreparedDriftCampaign> {
const servers = driftTargets([server]);
const entries = await scanServerDrift(server);
const files: PreparedDriftCampaign['files'] = [];
for (const entry of entries) {
const before = await readExistingFile(join(destOfKind(server, entry.kind), entry.relPath));
const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(server, entry);
if (entry.driftKind !== 'untracked' && !after) throw new Error(`Cannot resolve promoted content for ${entry.relPath}`);
files.push({ before, after, file: { server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind,
beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null } });
}
await requireServerDirectory(server);
if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
return { plan: buildPlan(servers, files.map(({ file }) => file)), files };
}
/** Record the audited plan atomically with a queued rebuild, never a second restart. */
export function createDriftRebuildCampaign(
server: string, operationId: string, incidentId: string | number, prepared: PreparedDriftCampaign
): string | null {
if (!prepared.files.length) return null;
if (prepared.plan.servers.length !== 1 || prepared.plan.servers[0] !== server
|| prepared.files.some(({ file }) => file.server !== server)) throw new Error('Drift evidence belongs to another server');
if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
return db().transaction(() => {
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server = ? AND action = 'rebuild'").get(operationId, server)) {
throw new Error('Drift campaign requires the matching rebuild operation');
}
const id = deploymentIdNow();
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')")
.run(id, new Date().toISOString());
const insertFile = db().prepare(`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
for (const { file, before, after } of prepared.files) {
insertFile.run(id, server, after === null ? 'removed' : before === null ? 'added' : 'updated',
file.relPath, file.beforeSha256, file.afterSha256, before, after);
}
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id)
VALUES (?, ?, ?, 'pending', ?)`).run(id, server, prepared.files.length, operationId);
db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)")
.run(operationId, id, server);
db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
.run(id, `Watchdog ${incidentId}: captured drift evidence; rebuild ${operationId} will install the latest promoted setup. Files remain unchanged until provisioning rebuilds the instance.`);
emit({ type: 'deployment-opened', deploymentId: id, action: 'drift', servers: [server] });
return id;
})();
}
/**
* Detect drift on the requested servers (or the whole fleet) and either
* report the plan (dryRun) or stage the changes as a single deployment.
* Provisioning applies them and restarts each affected server once it's empty.
*/
export async function runDriftDeployment(input: RunDriftInput = {}): Promise<RunDriftResult> {
const { dryRun = false, triggeredBy = 'ui' } = input;
const scanTargets = driftTargets(input.servers);
const perServer: Array<{
server: string;
removed: string[];
restored: string[];
skipped: string[];
entries: DriftEntry[];
}> = [];
for (const name of scanTargets) {
const entries = await scanServerDrift(name);
perServer.push({ server: name, removed: [], restored: [], skipped: [], entries });
}
// Recheck identities, existing bytes, and replacement bytes before any write.
const plan = input.expectedFingerprint ? await describePlan(scanTargets, perServer.flatMap(server => server.entries)) : null;
if (plan && plan.fingerprint !== input.expectedFingerprint) throw new DriftPlanChanged(plan);
const totalDrift = perServer.reduce((sum, s) => sum + s.entries.length, 0);
if (totalDrift === 0 || dryRun) {
return {
deploymentId: null,
perServer: perServer.map((s) => ({
server: s.server,
removed: s.entries.filter((e) => e.driftKind === 'untracked').map((e) => e.relPath),
restored: s.entries.filter((e) => e.driftKind !== 'untracked').map((e) => e.relPath),
skipped: []
})),
totalRemoved: perServer.reduce(
(n, s) => n + s.entries.filter((e) => e.driftKind === 'untracked').length,
0
),
totalRestored: perServer.reduce(
(n, s) => n + s.entries.filter((e) => e.driftKind !== 'untracked').length,
0
),
totalServers: perServer.filter((s) => s.entries.length > 0).length,
dryRun
};
}
// Commit. Create the deployment row FIRST so audit rows (which FK to it)
// can be inserted as we go.
const depId = deploymentIdNow();
const now = new Date().toISOString();
db().prepare(
`INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, ?)`
).run(depId, now, triggeredBy);
const insertServer = db().prepare(
`INSERT INTO deployment_servers
(deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error)
VALUES (?, ?, ?, 'pending', NULL, NULL, NULL, 0, NULL)`
);
const insertFile = db().prepare(
`INSERT INTO deployment_files
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
);
const affected: string[] = [];
for (const s of perServer) {
if (s.entries.length === 0) continue;
if (hasActiveBeta(s.server)) {
s.skipped.push('Active beta owns this server; reconciliation deferred');
continue;
}
for (const entry of s.entries) {
const dest = destOfKind(s.server, entry.kind);
const abs = join(dest, entry.relPath);
try {
await requireServerDirectory(s.server);
if (hasActiveBeta(s.server)) {
s.skipped.push(`${entry.relPath}: active beta owns this server; reconciliation deferred`);
continue;
}
const reviewed = plan?.files.find(file => file.server === s.server && file.kind === entry.kind && file.relPath === entry.relPath);
if (reviewed) {
const current = await readFile(abs).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; });
if ((current ? sha256Buffer(current) : null) !== reviewed.beforeSha256) {
s.skipped.push(`${entry.relPath}: changed after review; scan and review again`);
continue;
}
}
if (entry.driftKind === 'untracked') {
// Capture the reviewed deletion without touching the live tree.
const content = await readFile(abs);
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
db().transaction(() => {
const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, content, null);
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
kind: entry.kind, relPath: entry.relPath, content: null,
inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content) });
})();
s.removed.push(entry.relPath);
} else {
// Persist exact replacement bytes so waiting cannot change the plan.
const written = await resolveManagedEntry(s.server, entry);
if (!written) {
s.skipped.push(`${entry.relPath}: missing artifact could not be resolved from the DB`);
continue;
}
if (reviewed && sha256Buffer(written) !== reviewed.afterSha256) {
s.skipped.push(`${entry.relPath}: promoted content changed after review; scan and review again`);
continue;
}
const before = await readFile(abs).catch((err: NodeJS.ErrnoException) => {
if (err.code === 'ENOENT') return null;
throw err;
});
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
db().transaction(() => {
const file = insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath,
before ? sha256Buffer(before) : null, sha256Buffer(written), before, written);
queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
kind: entry.kind, relPath: entry.relPath, content: written,
scriptVersionId: entry.kind === 'config' ? null : entry.scriptVersionId ?? null,
inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
expectedBeforeSha: reviewed ? reviewed.beforeSha256 : before ? sha256Buffer(before) : null });
})();
s.restored.push(entry.relPath);
}
} catch (err) {
s.skipped.push(`${entry.relPath}: ${(err as Error).message}`);
}
}
const changes = s.removed.length + s.restored.length;
if (changes > 0) {
insertServer.run(depId, s.server, changes);
affected.push(s.server);
}
}
if (affected.length === 0) {
// Everything failed. Roll back the empty deployment row so it doesn't
// dangle in the UI as a mysterious no-op.
db().prepare('DELETE FROM deployments WHERE id = ?').run(depId);
return {
deploymentId: null,
perServer: perServer.map((s) => ({
server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped
})),
totalRemoved: 0,
totalRestored: 0,
totalServers: 0,
dryRun: false
};
}
emit({
type: 'deployment-opened',
deploymentId: depId,
action: 'drift',
servers: affected
});
return {
deploymentId: depId,
perServer: perServer.map((s) => ({
server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped
})),
totalRemoved: perServer.reduce((sum, s) => sum + s.removed.length, 0),
totalRestored: perServer.reduce((sum, s) => sum + s.restored.length, 0),
totalServers: affected.length,
dryRun: false
};
}