feat: add Watchdog recovery and drift rebuild workflows
This commit is contained in:
+129
-37
@@ -1,22 +1,20 @@
|
||||
import { mkdir, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { dirname, join, posix, relative, sep } from 'node:path';
|
||||
import { configFileOf, destOfKind, type ArtifactKind } from '../config.js';
|
||||
import { configFileOf, destOfKind, serverDir, type ArtifactKind } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { emit } from '../pipeline/bus.js';
|
||||
import { sha256Buffer } from './files.js';
|
||||
import {
|
||||
getServerScript,
|
||||
listServerScripts,
|
||||
recordScript,
|
||||
removeScript
|
||||
} from './inventory.js';
|
||||
import { getBaselineForScope, getVersion } from './scripts.js';
|
||||
import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
|
||||
import { selectedZombieMap } from './map-aliases.js';
|
||||
import { serverDirectories, getServer, listServers } from './servers.js';
|
||||
import { getServer, listServers } from './servers.js';
|
||||
import { promotedArtifacts } from './promoted-artifacts.js';
|
||||
|
||||
// Destinations drift scans, in a stable order so the UI shows scripts first.
|
||||
@@ -54,14 +52,35 @@ function toPosix(p: string): string {
|
||||
return p.split(sep).join(posix.sep);
|
||||
}
|
||||
|
||||
function hasActiveBeta(server: string): boolean {
|
||||
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
|
||||
WHERE bs.server = ? AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server);
|
||||
}
|
||||
|
||||
async function requireServerDirectory(server: string): Promise<void> {
|
||||
try {
|
||||
if (!(await stat(serverDir(server))).isDirectory()) throw new Error('path is not a directory');
|
||||
} catch (error) {
|
||||
throw new Error(`Server files unavailable for ${server}: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function readExistingFile(path: string): Promise<Buffer | null> {
|
||||
return readFile(path).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === 'ENOENT') return null;
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
|
||||
async function walkFiles(root: string): Promise<string[]> {
|
||||
const out: string[] = [];
|
||||
async function recur(dir: string): Promise<void> {
|
||||
let entries;
|
||||
try {
|
||||
entries = await readdir(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return;
|
||||
throw error;
|
||||
}
|
||||
for (const e of entries) {
|
||||
if (DRIFT_IGNORE_FILENAMES.has(e.name)) continue;
|
||||
@@ -76,27 +95,28 @@ async function walkFiles(root: string): Promise<string[]> {
|
||||
|
||||
/**
|
||||
* Scan every managed destination tree on a server (scripts, configs, plugins)
|
||||
* and return every file that isn't tracked in server_scripts. During a beta,
|
||||
* the beta bytes live at the actual target filename and the inventory row is
|
||||
* updated to reflect them, so beta files register as tracked with no
|
||||
* special-casing (pre-beta bytes are snapshotted to beta_file_snapshots for
|
||||
* rollback rather than left on disk as .OLD).
|
||||
*
|
||||
* The inventory table (server_scripts) keys on (server, rel_path) with no
|
||||
* destination discriminator. Rel_paths across destinations don't collide in
|
||||
* practice (`.gsc` for scripts, `.cfg` for configs, `.dll` for plugins), so
|
||||
* a shared tracked-set is safe.
|
||||
* and compare disk contents with promoted artifacts and inventory. An active
|
||||
* beta owns its server's files until promotion or rollback; defer the whole
|
||||
* server so reconciliation cannot replace experimental config or missing beta
|
||||
* artifacts with the promoted versions.
|
||||
*/
|
||||
export async function scanServerDrift(server: string): Promise<DriftEntry[]> {
|
||||
const tracked = new Set(listServerScripts(server).map((r) => r.rel_path));
|
||||
const drift = await scanManagedForServer(server);
|
||||
const managed = new Set(drift.map((entry) => `${entry.kind}:${entry.relPath}`));
|
||||
const srv = getServer(server);
|
||||
if (!srv || srv.destroyed_at || hasActiveBeta(server)) return [];
|
||||
// A missing mount is unavailable evidence, not an empty managed tree.
|
||||
await requireServerDirectory(server);
|
||||
const drift = await scanManagedForServer(server);
|
||||
// Previous inventory is an audit of installed bytes, not permission to
|
||||
// retain artifacts whose scope was moved or removed from the library.
|
||||
const managed = new Set(promotedArtifacts(srv.game, srv.map)
|
||||
.map(({ kind, script }) => `${kind}:${script.target_rel_path}`));
|
||||
const baseline = srv.game ? getBaselineForScope('game', srv.game) : undefined;
|
||||
if (baseline?.current_version_id) managed.add(`config:${configFileOf(server)}`);
|
||||
// Without a promoted baseline, Plutainer owns the seeded config. Keep it
|
||||
// until a baseline is promoted; startup arguments handle map selection.
|
||||
const keepDefaultConfig = srv?.game
|
||||
&& selectedZombieMap(srv.game, srv.map, srv.map_slug)
|
||||
&& !getBaselineForScope('game', srv.game)?.current_version_id;
|
||||
&& !baseline?.current_version_id;
|
||||
|
||||
// Scan for untracked files only after resolving managed targets.
|
||||
for (const kind of DRIFT_DESTS) {
|
||||
@@ -105,7 +125,7 @@ export async function scanServerDrift(server: string): Promise<DriftEntry[]> {
|
||||
if (onDisk.length === 0) continue;
|
||||
|
||||
for (const relPath of onDisk) {
|
||||
if (tracked.has(relPath) || managed.has(`${kind}:${relPath}`)) continue;
|
||||
if (managed.has(`${kind}:${relPath}`)) continue;
|
||||
if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server)) continue;
|
||||
if (kind === 'config' && relPath === MAP_CONFIG_FILE && srv?.created_by
|
||||
&& srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) continue;
|
||||
@@ -120,13 +140,15 @@ export async function scanServerDrift(server: string): Promise<DriftEntry[]> {
|
||||
kind,
|
||||
driftKind: 'untracked'
|
||||
});
|
||||
} catch {
|
||||
/* file disappeared between walk and stat — skip */
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
// A file disappeared between walk and stat; the next scan sees it.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return drift;
|
||||
await requireServerDirectory(server);
|
||||
return hasActiveBeta(server) ? [] : drift;
|
||||
}
|
||||
|
||||
// Every promoted artifact whose scope matches this server's game/map should
|
||||
@@ -143,21 +165,23 @@ async function scanManagedForServer(server: string): Promise<DriftEntry[]> {
|
||||
? getServerMapConfig(srv.game, srv.map, srv.map_slug) : null;
|
||||
if (mapConfig) {
|
||||
const abs = join(destOfKind(server, 'config'), MAP_CONFIG_FILE);
|
||||
const content = await readFile(abs).catch(() => null);
|
||||
const content = await readExistingFile(abs);
|
||||
if (!content?.equals(mapConfig)) {
|
||||
out.push({ server, relPath: MAP_CONFIG_FILE, kind: 'config', driftKind: content ? 'replace' : 'missing',
|
||||
size: mapConfig.byteLength, sha256: sha256Buffer(mapConfig) });
|
||||
}
|
||||
}
|
||||
|
||||
// Existing files without inventory must be installed/tracked in one pass,
|
||||
// never deleted first. Preserve tracked files (including beta versions).
|
||||
// Inventory records the last managed write, not the current disk bytes.
|
||||
// Check both so edits and outdated versions are detected on tracked files.
|
||||
for (const { kind, script, version } of promotedArtifacts(srv.game, srv.map)) {
|
||||
const present = existsSync(join(destOfKind(server, kind), script.target_rel_path));
|
||||
if (present && getServerScript(server, script.target_rel_path)) continue;
|
||||
const content = await readExistingFile(join(destOfKind(server, kind), script.target_rel_path));
|
||||
const inventory = getServerScript(server, script.target_rel_path);
|
||||
if (content?.equals(version.content) && inventory?.sha256 === version.sha256
|
||||
&& inventory.script_version_id === version.id) continue;
|
||||
out.push({
|
||||
server, relPath: script.target_rel_path, kind,
|
||||
driftKind: present ? 'replace' : 'missing',
|
||||
driftKind: content ? 'replace' : 'missing',
|
||||
size: version.size, sha256: version.sha256,
|
||||
scriptId: script.id, scriptVersionId: version.id
|
||||
});
|
||||
@@ -176,13 +200,14 @@ async function scanManagedForServer(server: string): Promise<DriftEntry[]> {
|
||||
const targetRel = configFileOf(server);
|
||||
const abs = join(dest, targetRel);
|
||||
const inv = getServerScript(server, targetRel);
|
||||
const onDiskMatches = existsSync(abs) && inv?.sha256 === composed.sha256;
|
||||
const content = await readExistingFile(abs);
|
||||
const onDiskMatches = content?.equals(composed.content) && inv?.sha256 === composed.sha256;
|
||||
if (!onDiskMatches) {
|
||||
out.push({
|
||||
server,
|
||||
relPath: targetRel,
|
||||
kind: 'config',
|
||||
driftKind: existsSync(abs) ? 'replace' : 'missing',
|
||||
driftKind: content ? 'replace' : 'missing',
|
||||
size: composed.content.byteLength,
|
||||
sha256: composed.sha256,
|
||||
scriptId: baseline.id,
|
||||
@@ -230,12 +255,8 @@ function composeExpectedCfg(
|
||||
*/
|
||||
export async function scanFleetDrift(): Promise<DriftEntry[]> {
|
||||
const servers = listServers().map((s) => s.name);
|
||||
const registered = new Set(servers);
|
||||
const allDirs = new Set(await serverDirectories());
|
||||
// Only touch server dirs that are both registered AND present on disk.
|
||||
const eligible = servers.filter((n) => registered.has(n) && allDirs.has(n));
|
||||
const out: DriftEntry[] = [];
|
||||
for (const name of eligible) {
|
||||
for (const name of servers) {
|
||||
const rows = await scanServerDrift(name);
|
||||
out.push(...rows);
|
||||
}
|
||||
@@ -271,6 +292,10 @@ async function describePlan(servers: string[], entries: DriftEntry[]): Promise<D
|
||||
beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
|
||||
beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null });
|
||||
}
|
||||
return buildPlan(servers, files);
|
||||
}
|
||||
|
||||
function buildPlan(servers: string[], files: DriftPlanFile[]): DriftPlan {
|
||||
files.sort((a, b) => `${a.server}:${a.kind}:${a.relPath}`.localeCompare(`${b.server}:${b.kind}:${b.relPath}`));
|
||||
return { servers, files, scannedAt: new Date().toISOString(), fingerprint: sha256Buffer(Buffer.from(JSON.stringify({ servers, files }))) };
|
||||
}
|
||||
@@ -298,6 +323,62 @@ function deploymentIdNow(): string {
|
||||
return `dep-${stamp}-drift-${randomUUID().slice(0, 8)}`;
|
||||
}
|
||||
|
||||
export interface PreparedDriftCampaign {
|
||||
plan: DriftPlan;
|
||||
files: Array<{ file: DriftPlanFile; before: Buffer | null; after: Buffer | null }>;
|
||||
}
|
||||
|
||||
/** Capture a rebuild's file evidence without changing the running instance. */
|
||||
export async function prepareDriftCampaign(server: string): Promise<PreparedDriftCampaign> {
|
||||
const servers = driftTargets([server]);
|
||||
const entries = await scanServerDrift(server);
|
||||
const files: PreparedDriftCampaign['files'] = [];
|
||||
for (const entry of entries) {
|
||||
const before = await readExistingFile(join(destOfKind(server, entry.kind), entry.relPath));
|
||||
const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(server, entry);
|
||||
if (entry.driftKind !== 'untracked' && !after) throw new Error(`Cannot resolve promoted content for ${entry.relPath}`);
|
||||
files.push({ before, after, file: { server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind,
|
||||
beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
|
||||
beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null } });
|
||||
}
|
||||
await requireServerDirectory(server);
|
||||
if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
|
||||
return { plan: buildPlan(servers, files.map(({ file }) => file)), files };
|
||||
}
|
||||
|
||||
/** Record the audited plan atomically with a queued rebuild, never a second restart. */
|
||||
export function createDriftRebuildCampaign(
|
||||
server: string, operationId: string, incidentId: string | number, prepared: PreparedDriftCampaign
|
||||
): string | null {
|
||||
if (!prepared.files.length) return null;
|
||||
if (prepared.plan.servers.length !== 1 || prepared.plan.servers[0] !== server
|
||||
|| prepared.files.some(({ file }) => file.server !== server)) throw new Error('Drift evidence belongs to another server');
|
||||
if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
|
||||
return db().transaction(() => {
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server = ? AND action = 'rebuild'").get(operationId, server)) {
|
||||
throw new Error('Drift campaign requires the matching rebuild operation');
|
||||
}
|
||||
const id = deploymentIdNow();
|
||||
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')")
|
||||
.run(id, new Date().toISOString());
|
||||
const insertFile = db().prepare(`INSERT INTO deployment_files
|
||||
(deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
|
||||
for (const { file, before, after } of prepared.files) {
|
||||
insertFile.run(id, server, after === null ? 'removed' : before === null ? 'added' : 'updated',
|
||||
file.relPath, file.beforeSha256, file.afterSha256, before, after);
|
||||
}
|
||||
db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id)
|
||||
VALUES (?, ?, ?, 'pending', ?)`).run(id, server, prepared.files.length, operationId);
|
||||
db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)")
|
||||
.run(operationId, id, server);
|
||||
db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
|
||||
.run(id, `Watchdog ${incidentId}: captured drift evidence; rebuild ${operationId} will install the latest promoted setup. Files remain unchanged until provisioning rebuilds the instance.`);
|
||||
emit({ type: 'deployment-opened', deploymentId: id, action: 'drift', servers: [server] });
|
||||
return id;
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect drift on the requested servers (or the whole fleet) and either
|
||||
* report the plan (dryRun) or remove untracked files and install managed
|
||||
@@ -368,10 +449,19 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
|
||||
const affected: string[] = [];
|
||||
for (const s of perServer) {
|
||||
if (s.entries.length === 0) continue;
|
||||
if (hasActiveBeta(s.server)) {
|
||||
s.skipped.push('Active beta owns this server; reconciliation deferred');
|
||||
continue;
|
||||
}
|
||||
for (const entry of s.entries) {
|
||||
const dest = destOfKind(s.server, entry.kind);
|
||||
const abs = join(dest, entry.relPath);
|
||||
try {
|
||||
await requireServerDirectory(s.server);
|
||||
if (hasActiveBeta(s.server)) {
|
||||
s.skipped.push(`${entry.relPath}: active beta owns this server; reconciliation deferred`);
|
||||
continue;
|
||||
}
|
||||
const reviewed = plan?.files.find(file => file.server === s.server && file.kind === entry.kind && file.relPath === entry.relPath);
|
||||
if (reviewed) {
|
||||
const current = await readFile(abs).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; });
|
||||
@@ -383,6 +473,7 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
|
||||
if (entry.driftKind === 'untracked') {
|
||||
// Delete a file inventory doesn't know about.
|
||||
const content = await readFile(abs);
|
||||
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
|
||||
await rm(abs, { force: true });
|
||||
removeScript(s.server, entry.relPath);
|
||||
s.removed.push(entry.relPath);
|
||||
@@ -403,6 +494,7 @@ export async function runDriftDeployment(input: RunDriftInput = {}): Promise<Run
|
||||
throw err;
|
||||
});
|
||||
await mkdir(dirname(abs), { recursive: true });
|
||||
if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
|
||||
await writeFile(abs, written);
|
||||
recordScript(
|
||||
s.server,
|
||||
|
||||
Reference in New Issue
Block a user