electric-cherry/src/lib/server/fleet/drift.ts

import { readdir, readFile, stat } from 'node:fs/promises';
import { randomUUID } from 'node:crypto';
import type { DriftPlan, DriftPlanFile } from '$lib/drift-plan.js';
import { join, posix, relative, sep } from 'node:path';
import { configFileOf, destOfKind, serverDir, type ArtifactKind } from '../config.js';
import { db } from '../db/index.js';
import { emit } from '../pipeline/bus.js';
import { sha256Buffer } from './files.js';
import { getServerScript } from './inventory.js';
import { getBaselineForScope, getVersion } from './scripts.js';
import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
import { selectedZombieMap } from './map-aliases.js';
import { getServer, listServers } from './servers.js';
import { promotedArtifacts } from './promoted-artifacts.js';
import { queueFileChange } from './staged-files.js';

// Destinations drift scans, in a stable order so the UI shows scripts first.
// Baseline shares the config destination — one physical dir, two logical kinds.
const DRIFT_DESTS: ReadonlyArray<ArtifactKind> = ['script', 'config', 'plugin'];

// Filenames we never treat as drift. dedicated.cfg is placed by Plutonium for
// multiplayer and regenerates if deleted; we manage dedicated_sp.cfg (zombies)
// via config-compose instead.
const DRIFT_IGNORE_FILENAMES = new Set(['dedicated.cfg']);

// Three flavors of drift:
//   'untracked' — file outside inventory and promoted scope (delete it)
//   'replace'   — managed target exists but needs installation/tracking
//   'missing'   — artifact promoted for this server's scope, but file is gone
//                 (write it back from the DB)
export type DriftKind = 'untracked' | 'missing' | 'replace';

export interface DriftEntry {
    server: string;
    relPath: string;
    kind: ArtifactKind;
    driftKind: DriftKind;
    // Populated for on-disk detections (untracked) and missing configs where
    // we compute the expected content.
    size: number;
    sha256: string;
    // Populated for missing/replacement entries so runDriftDeployment can write the
    // right content back.
    scriptId?: number;
    scriptVersionId?: number | null;
}

function toPosix(p: string): string {
    return p.split(sep).join(posix.sep);
}

function hasActiveBeta(server: string): boolean {
    return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
        WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server);
}

async function requireServerDirectory(server: string): Promise<void> {
    try {
        if (!(await stat(serverDir(server))).isDirectory()) throw new Error('path is not a directory');
    } catch (error) {
        throw new Error(`Server files unavailable for ${server}: ${(error as Error).message}`);
    }
}

async function readExistingFile(path: string): Promise<Buffer | null> {
    return readFile(path).catch((error: NodeJS.ErrnoException) => {
        if (error.code === 'ENOENT') return null;
        throw error;
    });
}

async function walkFiles(root: string): Promise<string[]> {
    const out: string[] = [];
    async function recur(dir: string): Promise<void> {
        let entries;
        try {
            entries = await readdir(dir, { withFileTypes: true });
        } catch (error) {
            if ((error as NodeJS.ErrnoException).code === 'ENOENT') return;
            throw error;
        }
        for (const e of entries) {
            if (DRIFT_IGNORE_FILENAMES.has(e.name)) continue;
            const abs = join(dir, e.name);
            if (e.isDirectory()) await recur(abs);
            else if (e.isFile()) out.push(toPosix(relative(root, abs)));
        }
    }
    await recur(root);
    return out.sort();
}

/**
 * Scan every managed destination tree on a server (scripts, configs, plugins)
 * and compare disk contents with promoted artifacts and inventory. An active
 * beta owns its server's files until promotion or rollback; defer the whole
 * server so reconciliation cannot replace experimental config or missing beta
 * artifacts with the promoted versions.
 */
export async function scanServerDrift(server: string): Promise<DriftEntry[]> {
    const srv = getServer(server);
    if (!srv || srv.destroyed_at || hasActiveBeta(server)) return [];
    // A missing mount is unavailable evidence, not an empty managed tree.
    await requireServerDirectory(server);
    const drift = await scanManagedForServer(server);
    // Previous inventory is an audit of installed bytes, not permission to
    // retain artifacts whose scope was moved or removed from the library.
    const managed = new Set(promotedArtifacts(srv.game, srv.map)
        .map(({ kind, script }) => `${kind}:${script.target_rel_path}`));
    const baseline = srv.game ? getBaselineForScope('game', srv.game) : undefined;
    if (baseline?.current_version_id) managed.add(`config:${configFileOf(server)}`);
    // Without a promoted baseline, Plutainer owns the seeded config. Keep it
    // until a baseline is promoted; startup arguments handle map selection.
    const keepDefaultConfig = srv?.game
        && selectedZombieMap(srv.game, srv.map, srv.map_slug)
        && !baseline?.current_version_id;

    // Scan for untracked files only after resolving managed targets.
    for (const kind of DRIFT_DESTS) {
        const dest = destOfKind(server, kind);
        const onDisk = await walkFiles(dest);
        if (onDisk.length === 0) continue;

        for (const relPath of onDisk) {
            if (managed.has(`${kind}:${relPath}`)) continue;
            if (keepDefaultConfig && kind === 'config' && relPath === configFileOf(server)) continue;
            if (kind === 'config' && relPath === MAP_CONFIG_FILE && srv?.created_by
                && srv.game_mode === 'zombie' && getServerMapConfig(srv.game, srv.map, srv.map_slug)) continue;
            const abs = join(dest, relPath);
            try {
                const st = await stat(abs);
                const buf = await readFile(abs);
                drift.push({
                    server, relPath,
                    size: st.size,
                    sha256: sha256Buffer(buf),
                    kind,
                    driftKind: 'untracked'
                });
            } catch (error) {
                if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
                // A file disappeared between walk and stat; the next scan sees it.
            }
        }
    }

    await requireServerDirectory(server);
    return hasActiveBeta(server) ? [] : drift;
}

// Every promoted artifact whose scope matches this server's game/map should
// have its target file present on disk. Anything absent is 'missing' drift.
// Configs are handled as a composed layer (baseline + overlays → one cfg).
async function scanManagedForServer(server: string): Promise<DriftEntry[]> {
    const srv = getServer(server);
    if (!srv) return [];
    const out: DriftEntry[] = [];

    // Provisioned servers load this generated file after their main cfg. It
    // remains managed even when there is no promoted baseline yet.
    const mapConfig = srv.created_by && srv.game_mode === 'zombie'
        ? getServerMapConfig(srv.game, srv.map, srv.map_slug) : null;
    if (mapConfig) {
        const abs = join(destOfKind(server, 'config'), MAP_CONFIG_FILE);
        const content = await readExistingFile(abs);
        if (!content?.equals(mapConfig)) {
            out.push({ server, relPath: MAP_CONFIG_FILE, kind: 'config', driftKind: content ? 'replace' : 'missing',
                size: mapConfig.byteLength, sha256: sha256Buffer(mapConfig) });
        }
    }

    // Inventory records the last managed write, not the current disk bytes.
    // Check both so edits and outdated versions are detected on tracked files.
    for (const { kind, script, version } of promotedArtifacts(srv.game, srv.map)) {
        const content = await readExistingFile(join(destOfKind(server, kind), script.target_rel_path));
        const inventory = getServerScript(server, script.target_rel_path);
        if (content?.equals(version.content) && inventory?.sha256 === version.sha256
            && inventory.script_version_id === version.id) continue;
        out.push({
            server, relPath: script.target_rel_path, kind,
            driftKind: content ? 'replace' : 'missing',
            size: version.size, sha256: version.sha256,
            scriptId: script.id, scriptVersionId: version.id
        });
    }

    // Composed cfg: the effective dedicated_sp.cfg is baseline + all promoted
    // overlays for this server's scope. It's "missing" when either the file
    // isn't on disk at all, or its sha256 differs from what we'd compose.
    // Only meaningful when there's a promoted baseline for this server's game.
    if (srv.game) {
        const baseline = getBaselineForScope('game', srv.game);
        if (baseline?.current_version_id) {
            const composed = composeExpectedCfg(srv.game, srv.map, srv.map_slug);
            if (composed) {
                const dest = destOfKind(server, 'config');
                const targetRel = configFileOf(server);
                const abs = join(dest, targetRel);
                const inv = getServerScript(server, targetRel);
                const content = await readExistingFile(abs);
                const onDiskMatches = content?.equals(composed.content) && inv?.sha256 === composed.sha256;
                if (!onDiskMatches) {
                    out.push({
                        server,
                        relPath: targetRel,
                        kind: 'config',
                        driftKind: content ? 'replace' : 'missing',
                        size: composed.content.byteLength,
                        sha256: composed.sha256,
                        scriptId: baseline.id,
                        scriptVersionId: baseline.current_version_id
                    });
                }
            }
        }
    }

    return out;
}

// Recompute what dedicated_sp.cfg SHOULD look like on this server given the
// current promoted baseline + overlays. Returns null if no baseline exists.
// Resolve the content we should install for a managed drift entry. For
// scripts/plugins, that's the current version's BLOB. For configs, recompose
// the effective cfg from baseline + overlays.
async function resolveManagedEntry(server: string, entry: DriftEntry): Promise<Buffer | null> {
    if (entry.kind === 'config') {
        const srv = getServer(server);
        if (!srv?.game) return null;
        if (entry.relPath === MAP_CONFIG_FILE) return getServerMapConfig(srv.game, srv.map, srv.map_slug);
        const composed = composeExpectedCfg(srv.game, srv.map, srv.map_slug);
        return composed?.content ?? null;
    }
    if (!entry.scriptVersionId) return null;
    const ver = getVersion(entry.scriptVersionId);
    return ver?.content ?? null;
}

function composeExpectedCfg(
    game: string | null,
    map: string | null,
    mapSlug?: string | null
): { content: Buffer; sha256: string } | null {
    const content = getPromotedServerConfig(game, map, mapSlug);
    if (!content) return null;
    return { content, sha256: sha256Buffer(content) };
}

/**
 * Scan the whole fleet. Servers are read from the DB (registered fleet), not
 * the on-disk SERVERS_ROOT listing, so unregistered folders don't leak in.
 */
export async function scanFleetDrift(): Promise<DriftEntry[]> {
    const servers = listServers().map((s) => s.name);
    const out: DriftEntry[] = [];
    for (const name of servers) {
        const rows = await scanServerDrift(name);
        out.push(...rows);
    }
    return out;
}

export interface RunDriftInput {
    servers?: string[];        // if omitted, scans the whole fleet
    dryRun?: boolean;
    triggeredBy?: 'ui' | 'sweep' | 'cron';
    expectedFingerprint?: string;
}

export class DriftPlanChanged extends Error {
    constructor(public plan: DriftPlan) { super('The files or target servers changed. Nothing was changed. Review the updated plan before continuing.'); }
}

function driftTargets(servers?: string[]): string[] {
    const registered = listServers().map(server => server.name);
    if (servers && (!servers.length || servers.some(name => !registered.includes(name)))) throw new Error('Select at least one registered server. Refresh the fleet if a server was removed.');
    return [...new Set(servers ?? registered)].sort();
}

async function describePlan(servers: string[], entries: DriftEntry[]): Promise<DriftPlan> {
    const files: DriftPlanFile[] = [];
    for (const entry of entries) {
        const before = await readFile(join(destOfKind(entry.server, entry.kind), entry.relPath)).catch((error: NodeJS.ErrnoException) => {
            if (error.code === 'ENOENT') return null;
            throw error;
        });
        const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(entry.server, entry);
        files.push({ server: entry.server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind,
            beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
            beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null });
    }
    return buildPlan(servers, files);
}

function buildPlan(servers: string[], files: DriftPlanFile[]): DriftPlan {
    files.sort((a, b) => `${a.server}:${a.kind}:${a.relPath}`.localeCompare(`${b.server}:${b.kind}:${b.relPath}`));
    return { servers, files, scannedAt: new Date().toISOString(), fingerprint: sha256Buffer(Buffer.from(JSON.stringify({ servers, files }))) };
}

export async function previewDrift(servers?: string[]): Promise<DriftPlan> {
    const targets = driftTargets(servers);
    const entries: DriftEntry[] = [];
    for (const server of targets) entries.push(...await scanServerDrift(server));
    return describePlan(targets, entries);
}

export interface RunDriftResult {
    deploymentId: string | null;
    // `removed` and `restored` describe queued changes. Provisioning applies
    // them only after the server passes its activation gates.
    perServer: Array<{ server: string; removed: string[]; restored: string[]; skipped: string[] }>;
    totalRemoved: number;
    totalRestored: number;
    totalServers: number;
    dryRun: boolean;
}

function deploymentIdNow(): string {
    const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
    return `dep-${stamp}-drift-${randomUUID().slice(0, 8)}`;
}

export interface PreparedDriftCampaign {
    plan: DriftPlan;
    files: Array<{ file: DriftPlanFile; before: Buffer | null; after: Buffer | null }>;
}

/** Capture a rebuild's file evidence without changing the running instance. */
export async function prepareDriftCampaign(server: string): Promise<PreparedDriftCampaign> {
    const servers = driftTargets([server]);
    const entries = await scanServerDrift(server);
    const files: PreparedDriftCampaign['files'] = [];
    for (const entry of entries) {
        const before = await readExistingFile(join(destOfKind(server, entry.kind), entry.relPath));
        const after = entry.driftKind === 'untracked' ? null : await resolveManagedEntry(server, entry);
        if (entry.driftKind !== 'untracked' && !after) throw new Error(`Cannot resolve promoted content for ${entry.relPath}`);
        files.push({ before, after, file: { server, relPath: entry.relPath, kind: entry.kind, driftKind: entry.driftKind,
            beforeSha256: before ? sha256Buffer(before) : null, afterSha256: after ? sha256Buffer(after) : null,
            beforeSize: before?.byteLength ?? null, afterSize: after?.byteLength ?? null } });
    }
    await requireServerDirectory(server);
    if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
    return { plan: buildPlan(servers, files.map(({ file }) => file)), files };
}

/** Record the audited plan atomically with a queued rebuild, never a second restart. */
export function createDriftRebuildCampaign(
    server: string, operationId: string, incidentId: string | number, prepared: PreparedDriftCampaign
): string | null {
    if (!prepared.files.length) return null;
    if (prepared.plan.servers.length !== 1 || prepared.plan.servers[0] !== server
        || prepared.files.some(({ file }) => file.server !== server)) throw new Error('Drift evidence belongs to another server');
    if (hasActiveBeta(server)) throw new Error('Active beta owns this server; reconciliation deferred');
    return db().transaction(() => {
        if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server = ? AND action = 'rebuild'").get(operationId, server)) {
            throw new Error('Drift campaign requires the matching rebuild operation');
        }
        const id = deploymentIdNow();
        db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, 'watchdog')")
            .run(id, new Date().toISOString());
        const insertFile = db().prepare(`INSERT INTO deployment_files
            (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
        for (const { file, before, after } of prepared.files) {
            insertFile.run(id, server, after === null ? 'removed' : before === null ? 'added' : 'updated',
                file.relPath, file.beforeSha256, file.afterSha256, before, after);
        }
        db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, provisioning_id)
            VALUES (?, ?, ?, 'pending', ?)`).run(id, server, prepared.files.length, operationId);
        db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'campaign', ?, ?)")
            .run(operationId, id, server);
        db().prepare('INSERT INTO campaign_events (deployment_id, message) VALUES (?, ?)')
            .run(id, `Watchdog ${incidentId}: captured drift evidence; rebuild ${operationId} will install the latest promoted setup. Files remain unchanged until provisioning rebuilds the instance.`);
        emit({ type: 'deployment-opened', deploymentId: id, action: 'drift', servers: [server] });
        return id;
    })();
}

/**
 * Detect drift on the requested servers (or the whole fleet) and either
 * report the plan (dryRun) or stage the changes as a single deployment.
 * Provisioning applies them and restarts each affected server once it's empty.
 */
export async function runDriftDeployment(input: RunDriftInput = {}): Promise<RunDriftResult> {
    const { dryRun = false, triggeredBy = 'ui' } = input;

    const scanTargets = driftTargets(input.servers);

    const perServer: Array<{
        server: string;
        removed: string[];
        restored: string[];
        skipped: string[];
        entries: DriftEntry[];
    }> = [];
    for (const name of scanTargets) {
        const entries = await scanServerDrift(name);
        perServer.push({ server: name, removed: [], restored: [], skipped: [], entries });
    }

    // Recheck identities, existing bytes, and replacement bytes before any write.
    const plan = input.expectedFingerprint ? await describePlan(scanTargets, perServer.flatMap(server => server.entries)) : null;
    if (plan && plan.fingerprint !== input.expectedFingerprint) throw new DriftPlanChanged(plan);
    const totalDrift = perServer.reduce((sum, s) => sum + s.entries.length, 0);
    if (totalDrift === 0 || dryRun) {
        return {
            deploymentId: null,
            perServer: perServer.map((s) => ({
                server: s.server,
                removed: s.entries.filter((e) => e.driftKind === 'untracked').map((e) => e.relPath),
                restored: s.entries.filter((e) => e.driftKind !== 'untracked').map((e) => e.relPath),
                skipped: []
            })),
            totalRemoved: perServer.reduce(
                (n, s) => n + s.entries.filter((e) => e.driftKind === 'untracked').length,
                0
            ),
            totalRestored: perServer.reduce(
                (n, s) => n + s.entries.filter((e) => e.driftKind !== 'untracked').length,
                0
            ),
            totalServers: perServer.filter((s) => s.entries.length > 0).length,
            dryRun
        };
    }

    // Commit. Create the deployment row FIRST so audit rows (which FK to it)
    // can be inserted as we go.
    const depId = deploymentIdNow();
    const now = new Date().toISOString();
    db().prepare(
        `INSERT INTO deployments (id, action, started_at, triggered_by) VALUES (?, 'drift', ?, ?)`
    ).run(depId, now, triggeredBy);
    const insertServer = db().prepare(
        `INSERT INTO deployment_servers
         (deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error)
         VALUES (?, ?, ?, 'pending', NULL, NULL, NULL, 0, NULL)`
    );
    const insertFile = db().prepare(
        `INSERT INTO deployment_files
         (deployment_id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after)
         VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
    );

    const affected: string[] = [];
    for (const s of perServer) {
        if (s.entries.length === 0) continue;
        if (hasActiveBeta(s.server)) {
            s.skipped.push('Active beta owns this server; reconciliation deferred');
            continue;
        }
        for (const entry of s.entries) {
            const dest = destOfKind(s.server, entry.kind);
            const abs = join(dest, entry.relPath);
            try {
                await requireServerDirectory(s.server);
                if (hasActiveBeta(s.server)) {
                    s.skipped.push(`${entry.relPath}: active beta owns this server; reconciliation deferred`);
                    continue;
                }
                const reviewed = plan?.files.find(file => file.server === s.server && file.kind === entry.kind && file.relPath === entry.relPath);
                if (reviewed) {
                    const current = await readFile(abs).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return null; throw error; });
                    if ((current ? sha256Buffer(current) : null) !== reviewed.beforeSha256) {
                        s.skipped.push(`${entry.relPath}: changed after review; scan and review again`);
                        continue;
                    }
                }
                if (entry.driftKind === 'untracked') {
                    // Capture the reviewed deletion without touching the live tree.
                    const content = await readFile(abs);
                    if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
                    db().transaction(() => {
                        const file = insertFile.run(depId, s.server, 'removed', entry.relPath, sha256Buffer(content), null, content, null);
                        queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
                            kind: entry.kind, relPath: entry.relPath, content: null,
                            inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
                            expectedBeforeSha: reviewed?.beforeSha256 ?? sha256Buffer(content) });
                    })();
                    s.removed.push(entry.relPath);
                } else {
                    // Persist exact replacement bytes so waiting cannot change the plan.
                    const written = await resolveManagedEntry(s.server, entry);
                    if (!written) {
                        s.skipped.push(`${entry.relPath}: missing artifact could not be resolved from the DB`);
                        continue;
                    }
                    if (reviewed && sha256Buffer(written) !== reviewed.afterSha256) {
                        s.skipped.push(`${entry.relPath}: promoted content changed after review; scan and review again`);
                        continue;
                    }
                    const before = await readFile(abs).catch((err: NodeJS.ErrnoException) => {
                        if (err.code === 'ENOENT') return null;
                        throw err;
                    });
                    if (hasActiveBeta(s.server)) throw new Error('active beta owns this server; reconciliation deferred');
                    db().transaction(() => {
                        const file = insertFile.run(depId, s.server, before ? 'updated' : 'added', entry.relPath,
                            before ? sha256Buffer(before) : null, sha256Buffer(written), before, written);
                        queueFileChange({ sourceKind: 'campaign', sourceId: depId, server: s.server,
                            kind: entry.kind, relPath: entry.relPath, content: written,
                            scriptVersionId: entry.kind === 'config' ? null : entry.scriptVersionId ?? null,
                            inventorySource: `drift:${depId}`, deploymentFileId: Number(file.lastInsertRowid),
                            expectedBeforeSha: reviewed ? reviewed.beforeSha256 : before ? sha256Buffer(before) : null });
                    })();
                    s.restored.push(entry.relPath);
                }
            } catch (err) {
                s.skipped.push(`${entry.relPath}: ${(err as Error).message}`);
            }
        }
        const changes = s.removed.length + s.restored.length;
        if (changes > 0) {
            insertServer.run(depId, s.server, changes);
            affected.push(s.server);
        }
    }

    if (affected.length === 0) {
        // Everything failed. Roll back the empty deployment row so it doesn't
        // dangle in the UI as a mysterious no-op.
        db().prepare('DELETE FROM deployments WHERE id = ?').run(depId);
        return {
            deploymentId: null,
            perServer: perServer.map((s) => ({
                server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped
            })),
            totalRemoved: 0,
            totalRestored: 0,
            totalServers: 0,
            dryRun: false
        };
    }

    emit({
        type: 'deployment-opened',
        deploymentId: depId,
        action: 'drift',
        servers: affected
    });

    return {
        deploymentId: depId,
        perServer: perServer.map((s) => ({
            server: s.server, removed: s.removed, restored: s.restored, skipped: s.skipped
        })),
        totalRemoved: perServer.reduce((sum, s) => sum + s.removed.length, 0),
        totalRestored: perServer.reduce((sum, s) => sum + s.restored.length, 0),
        totalServers: affected.length,
        dryRun: false
    };
}