electric-cherry/src/lib/server/provisioning/instances.ts

import { fleetSettings } from '../settings.js';
import { probePlayerCount, probeServerStatus } from '../fleet/status-evidence.js';
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
import { randomUUID } from 'node:crypto';
import type { LaunchPreview } from '$lib/deployment-plan.js';
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
import { mkdir, chmod, rm, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js';
import { db } from '../db/index.js';
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
import { getServer } from '../fleet/servers.js';
import { zombieMapBySlug, type GameMode, type ZombieMapEntry, type ZombieGame } from '../fleet/map-aliases.js';
import {
    komodoCreateDeployment,
    komodoDeploy,
    komodoDestroyDeployment,
    komodoDeleteDeployment,
    komodoInspectHealthy,
    komodoEnsureTag,
    komodoUpdateTagsOnResource,
    type CreateDeploymentSpec,
    type DeploymentEnvVar
} from '../integrations/komodo.js';
import { MAP_CONFIG_FILE } from '../fleet/server-config.js';
import { recordScript } from '../fleet/inventory.js';
import { getVersion } from '../fleet/scripts.js';

export type CreateServerFailure =
    | 'bad-request'
    | 'unconfigured'    // Komodo creds / target server missing
    | 'unknown-map'     // slug isn't in the catalog
    | 'no-slot'         // all lobby slots for this map are taken
    | 'no-port'         // port range exhausted
    | 'fs'              // couldn't create host dir
    | 'komodo'          // Komodo CreateDeployment / Deploy rejected
    | 'db';             // shouldn't happen, but wraps INSERT failures

export type CreateServerResult =
    | { ok: true; name: string; port: number; deploymentId: string }
    | { ok: false; kind: CreateServerFailure; message: string };

export type DestroyServerResult =
    | { ok: true; warnings: string[] }
    | { ok: false; kind: 'not-found' | 'conflict' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] };

export interface CreateServerInput {
    game?: ZombieGame;    // omitted by older clients: T5
    gameMode: GameMode;   // 'zombie' — only value supported today
    mapSlug: string;      // 'cosmodrome'
    plutoKey?: string;    // saved for this server ID when supplied
    expectedName?: string; // optional preview guard; never selects a lobby
    previewId?: string;
    createdBy?: string;   // 'ui' by default
}

/**
 * Find the lowest lobby number `n` such that `t5_<mode>_<slug>_<n>` isn't in
 * the servers table. Caps at 20 to stay inside the port range — with 20 total
 * ports across the whole fleet, no one map can hold more than that.
 */
export function nextLobbyNumber(baseName: string, additionalNames: string[] = []): number | null {
    const rows = db()
        .prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
            SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
            WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
        .all(`${baseName}_%`) as Array<{ name: string }>;
    const taken = new Set<number>();
    const prefix = `${baseName}_`;
    for (const r of [...rows, ...additionalNames.map(name => ({ name }))]) {
        if (!r.name.startsWith(prefix)) continue;
        const tail = r.name.slice(prefix.length);
        if (/^\d+$/.test(tail)) taken.add(Number.parseInt(tail, 10));
    }
    for (let n = 1; n <= 20; n++) {
        if (!taken.has(n)) return n;
    }
    return null;
}

/**
 * Lowest unused port in the configured range. Reads every server's game_port
 * from the DB (which EC controls end-to-end under the new naming scheme, so
 * the DB is authoritative — no need to ask Komodo).
 */
export function nextAvailablePort(additionalPorts: number[] = [], releasingServers: string[] = []): number | null {
    const rows = db()
        .prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
            SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
            WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
        .all() as Array<{ name: string; game_port: number }>;
    const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
    for (let p = config.portRange.min; p <= config.portRange.max; p++) {
        if (!taken.has(p)) return p;
    }
    return null;
}

function buildEnv(args: {
    game: ZombieGame;
    name: string;
    port: number;
    plutoKey: string;
}): DeploymentEnvVar[] {
    return [
        { variable: 'PLUTAINER_SERVER_NAME', value: args.name },
        { variable: 'PLUTAINER_PORT', value: String(args.port) },
        { variable: 'PLUTAINER_GAME', value: args.game === 't6' ? 't6zm' : 't5sp' },
        { variable: 'PLUTAINER_CONFIG_FILE', value: configFileForGame(args.game) },
        // Plutainer seeds/loads the main config first. Load our generated map
        // config afterward, then rotate. A separate exec avoids quoted values
        // in EXTRA_ARGS, which Plutainer splits on whitespace.
        { variable: 'PLUTAINER_MAP_ROTATE', value: 'false' },
        // Give telemetry scripts the registry name before the first map starts.
        { variable: 'PLUTAINER_EXTRA_ARGS', value: `+exec ${MAP_CONFIG_FILE} +set tws_server_id ${args.name} +map_rotate` },
        { variable: 'PLUTO_SERVER_KEY', value: args.plutoKey },
        { variable: 'PLUTAINER_RCON_PASSWORD', value: config.rcon.password }
    ];
}

function buildSpec(args: {
    game: ZombieGame;
    name: string;
    port: number;
    hostDir: string;
    plutoKey: string;
    recipe: LaunchRecipe;
}): CreateDeploymentSpec {
    return {
        name: args.name,
        server: config.komodo.targetServer,
        image: args.recipe.image,
        network: config.komodo.defaultNetwork,
        // Container listens on PLUTAINER_PORT (== args.port), so host and
        // container ports match — no NAT inside the bridge network.
        ports: [
            `${args.port}:${args.port}/udp`,
            `${args.port}:${args.port}/tcp`
        ],
        volumes: [
            `${args.hostDir}:/home/plutainer/app`,
            `${args.game === 't6' ? config.komodo.t6GameFilesVolume : config.komodo.gameFilesVolume}:/home/plutainer/gamefiles:ro`
        ],
        environment: buildEnv(args)
    };
}

let creating = false;
interface PreparedCreation {
    ok: true; name: string; port: number; plutoKey: string; mapEntry: ZombieMapEntry;
}

interface PreviewRow {
    id: string; expires_at: string; server: string | null; port: number | null;
    recipe: string; blueprint_name: string | null; deployment_id: string | null;
}

/** Read-only checks; a preview never reserves a slot or writes credentials. */
export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown }): LaunchPreview {
    if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.');
    let recipe: LaunchRecipe;
    let blueprintName: string | null = null;
    if (input.blueprintId !== undefined) {
        if (typeof input.blueprintId !== 'string') throw new Error('Invalid blueprint.');
        ({ recipe, blueprintName } = blueprintRecipe(input.blueprintId));
    } else if (input.deploymentId !== undefined) {
        if (typeof input.deploymentId !== 'string') throw new Error('Invalid deployment.');
        const source = recipeForDeployment(input.deploymentId);
        if (!source) throw new Error('This deployment has no recorded launch recipe.');
        blueprintName = source.blueprintName;
        // History describes the previous launch; only its map selection is reusable.
        recipe = captureRecipe(source.recipe.game, source.recipe.mapSlug);
    } else {
        if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.');
        recipe = captureRecipe(input.game, input.mapSlug);
    }
    const lobby = nextLobbyNumber(`${recipe.game}_zombie_${recipe.mapSlug}`);
    const name = lobby === null ? null : `${recipe.game}_zombie_${recipe.mapSlug}_${lobby}`;
    const port = nextAvailablePort();
    const configured = !!(config.komodo.address && config.komodo.apiKey && config.komodo.apiSecret && config.komodo.targetServer);
    const checks: LaunchPreview['checks'] = [
        { label: 'Komodo settings', state: configured ? 'pass' : 'blocked', detail: configured ? 'Connection settings and target host are configured.' : 'Configure Komodo credentials and a target host before launching.' },
        { label: 'Health verification', state: config.rcon.password ? 'pass' : 'blocked', detail: config.rcon.password ? 'RCON is configured. Container, RCON, and map checks run after launch.' : 'Configure RCON_PASSWORD so the new server can be verified.' },
        { label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All 20 lobbies for this map are occupied or reserved.' },
        { label: 'Game port', state: port === null ? 'blocked' : 'pass', detail: port === null ? 'No free game port. Free capacity before launching.' : `${port} / UDP and TCP. Availability is checked again at launch.` },
        { label: 'Promoted configuration', state: 'pass', detail: recipe.configSource },
        { label: 'Game files and host readiness', state: 'info', detail: 'The host must have the selected game and DLC files. This preview does not probe volumes, images, credentials, or network access.' }
    ];
    const id = randomUUID();
    const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
    db().prepare('DELETE FROM server_launch_previews WHERE expires_at < ? AND deployment_id IS NULL').run(new Date().toISOString());
    db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
        .run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
    return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName,
        checks, ready: checks.every(check => check.state !== 'blocked') };
}

// releasingServers is internal to atomic collection queueing; HTTP callers cannot set it.
export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult {
    return db().transaction((): CreateServerResult => {
        const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined;
        if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' };
        const recipe: LaunchRecipe | undefined = preview ? JSON.parse(preview.recipe) : undefined;
        if (recipe && (recipe.game !== (input.game ?? 't5') || recipe.mapSlug !== input.mapSlug || input.gameMode !== 'zombie')) {
            return { ok: false, kind: 'bad-request', message: 'The selection does not match the reviewed launch recipe.' };
        }
        // A repeated submit after a lost response returns the same operation.
        if (preview?.deployment_id) return { ok: true, name: preview.server!, port: preview.port!, deploymentId: preview.deployment_id };
        if (preview && preview.expires_at <= new Date().toISOString()) return { ok: false, kind: 'bad-request', message: 'Launch preview expired. Review a fresh preview.' };
        if (preview && (!preview.server || !preview.port || preview.server !== nextName(input) || preview.port !== nextAvailablePort([], releasingServers))) {
            return { ok: false, kind: 'no-slot', message: 'Available lobby or port changed. Refresh the preview and review the new server key.' };
        }
        if (recipe && !config.rcon.password) {
            return { ok: false, kind: 'unconfigured', message: 'Health-check settings changed. Refresh the preview.' };
        }
        const prepared = prepareCreation(input, releasingServers);
        if (!prepared.ok) return prepared;
        // A preview records choices, not version pins. Promotions remain authoritative.
        const currentRecipe = captureRecipe(input.game ?? 't5', prepared.mapEntry.slug);
        const id = beginDeployment(prepared.name, 'create', input.createdBy);
        db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, blueprint_name) VALUES (?, ?, ?)')
            .run(id, JSON.stringify(currentRecipe), preview?.blueprint_name ?? null);
        if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id = ? WHERE id = ?').run(id, preview.id);
        db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
        db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
            .run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical);
        deploymentStage(id, 'Queued for server creation');
        return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id };
    })();
}

/** Reserve the same identity and port before removing any part of the instance. */
export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServerResult {
    const server = getServer(name);
    const game = gameOfServer(name);
    const map = server?.map_slug && zombieMapBySlug(game, server.map_slug);
    if (!server || !map || !server.game_port || server.komodo_type !== 'deployment' || server.komodo_name !== name
        || (server.host_dir && server.host_dir !== serverDir(name))) {
        return { ok: false, kind: 'bad-request', message: 'Rebuild requires a managed server with its original map, directory, and port.' };
    }
    if (!config.komodo.targetServer || (!config.rcon.password && !triggeredBy.startsWith('watchdog:'))) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' };
    if (!getServerKey(name)) return { ok: false, kind: 'bad-request', message: `Save a server key for ${name} in Keys before rebuilding.` };
    // Validate that the replacement setup can be composed before destruction.
    const recipe = captureRecipe(game, map.slug);
    return db().transaction(() => {
        const id = beginDeployment(name, 'rebuild', triggeredBy);
        db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
        db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
            .run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
        db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
        db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe));
        deploymentStage(id, 'Waiting to rebuild the empty server');
        return { ok: true as const, name, port: server.game_port!, deploymentId: id };
    })();
}

/** Caller has just verified emptiness and excluded conflicting work. */
export async function rebuildServer(id: string): Promise<void> {
    const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
        WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
    if (!job) return;
    if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) {
        // The cleanup checkpoint must survive capacity waits and app restarts.
        if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding');
        return;
    }
    // Recheck prerequisites while the original instance still exists.
    if (!getServerKey(job.server)) { finishDeployment(id, { ok: false, message: 'Saved server key was removed; rebuild did not begin.' }); return; }
    try { captureRecipe(gameOfServer(job.server), job.map_slug); }
    catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); return; }
    db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
    destroying.add(job.server);
    try {
        const result = job.stage === REMOVING_SERVER
            ? await cleanupDestroyedServer(job.server, id) : await destroyServerUnlocked(job.server, id);
        if (!result.ok || result.warnings.length) {
            finishDeployment(id, { ok: false, message: result.ok ? 'Old server files could not be completely removed.' : result.message, warnings: result.warnings });
            return;
        }
        db().transaction(() => {
            db().prepare('UPDATE server_rebuild_jobs SET removed_at = ? WHERE deployment_id = ?').run(new Date().toISOString(), id);
            db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
            deploymentStage(id, 'Queued for server creation');
        })();
    } catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
    finally { destroying.delete(job.server); }
}

function nextName(input: CreateServerInput): string | null {
    const base = `${input.game ?? 't5'}_${input.gameMode}_${input.mapSlug}`;
    const lobby = nextLobbyNumber(base);
    return lobby === null ? null : `${base}_${lobby}`;
}

// Internal callers use the same durable queue and capacity limit as HTTP launches.
// Await this pass's provisioning when possible; a busy host leaves the job queued.
export async function createServer(input: CreateServerInput): Promise<CreateServerResult> {
    const queued = queueCreateServer(input);
    if (!queued.ok) return queued;
    const existing = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(queued.deploymentId) as { state: string };
    if (existing.state !== 'queued') return queued;
    const outcomes = await creationPass();
    const outcome = outcomes.get(queued.deploymentId);
    if (outcome instanceof Error) throw outcome;
    return outcome ?? queued;
}

function prepareCreation(input: CreateServerInput, releasingServers: string[] = []): PreparedCreation | Extract<CreateServerResult, { ok: false }> {
    const game = input.game ?? 't5';
    if (game !== 't5' && game !== 't6') {
        return { ok: false, kind: 'bad-request', message: 'Supported games are t5 and t6' };
    }
    if (input.gameMode !== 'zombie') {
        return { ok: false, kind: 'unknown-map', message: `unsupported game mode: ${input.gameMode}` };
    }
    if (!config.komodo.address || !config.komodo.apiKey || !config.komodo.apiSecret) {
        return { ok: false, kind: 'unconfigured', message: 'KOMODO_ADDRESS / KOMODO_API_KEY / KOMODO_API_SECRET must be set' };
    }
    if (!config.komodo.targetServer) {
        return { ok: false, kind: 'unconfigured', message: 'KOMODO_TARGET_SERVER must name the Komodo host that runs new deployments' };
    }

    const mapEntry = zombieMapBySlug(game, input.mapSlug);
    if (!mapEntry) {
        return { ok: false, kind: 'unknown-map', message: `no ${game} zombie map with slug '${input.mapSlug}'` };
    }

    const baseName = `${game}_${input.gameMode}_${mapEntry.slug}`;
    const lobby = nextLobbyNumber(baseName);
    if (lobby === null) {
        return { ok: false, kind: 'no-slot', message: `all lobby slots (1..20) for ${baseName} are taken` };
    }
    const name = `${baseName}_${lobby}`;
    if (input.expectedName !== undefined && input.expectedName !== name) {
        return { ok: false, kind: 'no-slot', message: 'Available lobby changed. Review the updated server ID and key, then deploy again.' };
    }
    const plutoKey = input.plutoKey?.trim() || getServerKey(name);
    if (!plutoKey) return { ok: false, kind: 'bad-request', message: `No saved key for ${name}. Enter a Plutonium server key.` };

    const port = nextAvailablePort([], releasingServers);
    if (port === null) {
        return {
            ok: false,
            kind: 'no-port',
            message: `no free port in ${config.portRange.min}..${config.portRange.max} — destroy an existing server first`
        };
    }

    // Save before external operations so a failed deployment can also be retried.
    try { saveServerKey(name, plutoKey); }
    catch (err) { return { ok: false, kind: 'bad-request', message: (err as Error).message }; }

    return { ok: true, name, port, plutoKey, mapEntry };
}

async function createServerUnlocked(input: CreateServerInput, prepared: PreparedCreation, operation: string): Promise<CreateServerResult> {
    const { name, port, plutoKey, mapEntry } = prepared;
    const game = gameOfServer(name);
    // Resolve once more at provisioning so promotions made while queued also apply.
    // Persist the exact attempted files for history, never as input to future launches.
    const recipe = captureRecipe(game, mapEntry.slug);
    db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)
        ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`)
        .run(operation, JSON.stringify(recipe));
    const configFile = recipe.configFile;
    deploymentStage(operation, 'Preparing server files');
    const hostDir = serverDir(name);

    // Step 1: filesystem. mkdir mode is masked by umask, so chmod explicitly
    // after — plutainer expects to be able to write to /home/plutainer/app.
    try {
        await mkdir(hostDir, { recursive: true, mode: 0o777 });
        await chmod(hostDir, 0o777);
        for (const file of recipe.files) {
            const dest = destOfKind(name, file.kind);
            const target = join(dest, file.path);
            await mkdir(dirname(target), { recursive: true, mode: 0o777 });
            // Make every newly created ancestor writable by Plutainer, even
            // when the manager's umask removed write permissions at mkdir.
            for (let dir = dirname(target); dir !== join(hostDir) && dir !== dirname(dir); dir = dirname(dir)) {
                await chmod(dir, 0o777);
            }
            await writeFile(target, Buffer.from(file.content, 'base64'));
            await chmod(target, 0o666);
        }
    } catch (err) {
        return { ok: false, kind: 'fs', message: `failed to create ${hostDir}: ${(err as Error).message}` };
    }

    // Step 2: Komodo CreateDeployment.
    deploymentStage(operation, 'Creating Komodo deployment');
    const spec = buildSpec({ game, name, port, hostDir, plutoKey, recipe });
    const createRes = await komodoCreateDeployment(spec);
    if (!createRes.ok) {
        await rm(hostDir, { recursive: true, force: true }).catch(() => {});
        return { ok: false, kind: 'komodo', message: `CreateDeployment: ${createRes.message}` };
    }

    // Step 3: DB row — insert *before* execute/Deploy so the sweep sees the
    // server immediately even if Deploy hangs.
    try {
        db().transaction(() => {
            db().prepare(
                `INSERT INTO servers
                    (name, komodo_type, komodo_name, game, map,
                     game_mode, map_slug, host_dir, config_file, game_id, image,
                     game_port, created_by, container_state)
                 VALUES
                    (?, 'deployment', ?, ?, ?,
                     ?, ?, ?, ?, ?, ?,
                     ?, ?, 'starting')`
            ).run(
                name, name, game, mapEntry.canonical,
                input.gameMode, mapEntry.slug, hostDir,
                configFile, game === 't6' ? 't6zm' : 't5sp', recipe.image,
                port, input.createdBy ?? 'ui'
            );
            for (const file of recipe.files) {
                const versionId = file.versionId && getVersion(file.versionId) ? file.versionId : null;
                recordScript(name, file.path, versionId, file.sha256, file.size, file.path === MAP_CONFIG_FILE ? 'create-map' : 'create');
            }
        })();
    } catch (err) {
        // Roll back Komodo + FS if the DB insert fails (very unlikely — most
        // often a race where the same name got inserted twice).
        await komodoDestroyDeployment(name).catch(() => {});
        await komodoDeleteDeployment(name).catch(() => {});
        await rm(hostDir, { recursive: true, force: true }).catch(() => {});
        return { ok: false, kind: 'db', message: `failed to insert server row: ${(err as Error).message}` };
    }

    // Step 4: apply Komodo tags so operators can filter the deployment list
    // by game/map. Best-effort — a tag failure doesn't roll back the server;
    // the deployment is fully functional without cosmetic tags. Fires in
    // parallel to keep the create latency down.
    void applyDeploymentTags(name, game, mapEntry.canonical);

    // Step 5: execute/Deploy — actually start the container.
    deploymentStage(operation, 'Starting container');
    db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operation);
    const deployRes = await komodoDeploy(name);
    if (!deployRes.ok) {
        if (deployRes.kind === 'unreachable' || (deployRes.httpStatus !== undefined && deployRes.httpStatus >= 500)) {
            // A timeout or server error may arrive after the remote boot began.
            // Preserve files, identity, and capacity until readiness is known.
            recordDeploymentEvent(operation, 'Startup response uncertain; verifying the existing server: ' + deployRes.message);
            return { ok: true, name, port, deploymentId: operation };
        }
        // Container never started — undo everything so retrying is clean.
        await komodoDeleteDeployment(name).catch(() => {});
        db().prepare('DELETE FROM servers WHERE name = ?').run(name);
        await rm(hostDir, { recursive: true, force: true }).catch(() => {});
        return { ok: false, kind: 'komodo', message: `Deploy: ${deployRes.message}` };
    }

    return { ok: true, name, port, deploymentId: operation };
}

/**
 * Ensure `game` and `map` tags exist in Komodo, then attach both to the given
 * deployment. Purely cosmetic — any failure here logs and returns; the caller
 * treats it as fire-and-forget.
 */
async function applyDeploymentTags(deployment: string, gameTagName: string, mapTagName: string): Promise<void> {
    try {
        const [gameTag, mapTag] = await Promise.all([
            komodoEnsureTag(gameTagName),
            komodoEnsureTag(mapTagName)
        ]);
        const tagIds: string[] = [];
        if (gameTag.ok) tagIds.push(gameTag.id);
        else console.warn(`[createServer] tag "${gameTagName}": ${gameTag.message}`);
        if (mapTag.ok) tagIds.push(mapTag.id);
        else console.warn(`[createServer] tag "${mapTagName}": ${mapTag.message}`);
        if (tagIds.length === 0) return;

        const res = await komodoUpdateTagsOnResource('Deployment', deployment, tagIds);
        if (!res.ok) {
            console.warn(`[createServer] UpdateResourceMeta (tags) for ${deployment}: ${res.message}`);
        }
    } catch (err) {
        console.warn(`[createServer] tag application for ${deployment} threw:`, err);
    }
}

const destroying = new Set<string>();

// Persist the request before returning so navigation and client disconnects cannot lose it.
function hasActiveCreation(name: string): boolean {
    return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name);
}

export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
    if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild')
        AND state IN ('waiting','queued','running')`).get(name)) {
        return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] };
    }
    const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
    if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
    const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
    if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id };
    if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] };
    const operation = db().transaction(() => {
        const id = beginDeployment(name, 'destroy', triggeredBy);
        db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
        deploymentStage(id, 'Waiting to verify the server is empty');
        return id;
    })();
    return { ok: true, warnings: [], deploymentId: operation };
}

let destructionTick: Promise<void> | null = null;
export function processDestructions(): Promise<void> {
    if (destructionTick) return destructionTick;
    destructionTick = runDestructions().finally(() => { destructionTick = null; });
    return destructionTick;
}

async function runDestructions(): Promise<void> {
    const queued = db().prepare(`SELECT id, server, stage, server_deleted_at FROM server_deployments WHERE action = 'destroy' AND state = 'waiting'
        AND (server_deleted_at IS NULL OR stage = ?) ORDER BY started_at`).all(REMOVING_SERVER) as { id: string; server: string; stage: string; server_deleted_at: string | null }[];
    for (const operation of queued) {
        if (creating || destroying.has(operation.server)) continue;
        if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
        destroying.add(operation.server);
        try {
            if (operation.stage === REMOVING_SERVER) {
                if (operation.server_deleted_at && getServer(operation.server)) {
                    finishDeployment(operation.id, { ok: false, message: 'A replacement server is registered; local cleanup cannot safely resume.' });
                    continue;
                }
                const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create'
                    AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id);
                if (newer) {
                    finishDeployment(operation.id, { ok: false, message: 'A newer creation exists; local cleanup cannot safely resume.' });
                    continue;
                }
                if (!claimRemoval(operation.id)) continue;
                finishDeployment(operation.id, await cleanupDestroyedServer(operation.server, operation.id));
                continue;
            }
            if (!getServer(operation.server)) {
                if (hasActiveCreation(operation.server)) {
                    deploymentStage(operation.id, 'Waiting for server creation to finish');
                    continue;
                }
                finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' });
                continue;
            }
            // A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
            let players = await probePlayerCount(operation.server);
            if (players === null) {
                const health = await komodoInspectHealthy(operation.server);
                if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
            }
            if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
            if (players !== 0) {
                deploymentStage(operation.id, players === null
                    ? 'Waiting for player count: server could not be reached'
                    : 'Waiting for server to empty: ' + players + ' player(s) online');
                continue;
            }
            if (!claimRemoval(operation.id)) continue;
            deploymentStage(operation.id, 'Server is empty (0 players verified)');
            const result = await destroyServerUnlocked(operation.server, operation.id);
            finishDeployment(operation.id, result);
        } catch (err) {
            const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string };
            if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message);
            else if (row.state !== 'cancelled') finishDeployment(operation.id, { ok: false, message: (err as Error).message });
        } finally { destroying.delete(operation.server); }
    }
}

async function destroyServerUnlocked(name: string, operation: string): Promise<DestroyServerResult> {
    const row = getServer(name);
    if (!row) {
        return { ok: false, kind: 'not-found', message: `no server named '${name}'`, warnings: [] };
    }

    const warnings: string[] = [];

    // 1: stop + remove the container. execute/DestroyDeployment queues the
    // teardown; the response comes back before the container is actually gone.
    deploymentStage(operation, 'Stopping and removing container');
    const destroyRes = await komodoDestroyDeployment(name);
    if (!destroyRes.ok) {
        return { ok: false, kind: 'komodo', message: 'DestroyDeployment failed: ' + destroyRes.message, warnings };
    }

    // 2: wait for the container to actually disappear. Komodo refuses to
    // DeleteDeployment while a container still exists for it, so poll the
    // health probe until it stops reporting a running state. `ok:false` here
    // is the *success* signal — it means the container is no longer inspectable.
    const pollDeadline = Date.now() + 30_000;
    while (Date.now() < pollDeadline) {
        const health = await komodoInspectHealthy(name);
        if (!health.ok) break;
        await new Promise((resolve) => setTimeout(resolve, 1500));
    }

    // 3: remove the deployment entity itself.
    deploymentStage(operation, 'Deleting Komodo deployment');
    const deleteRes = await komodoDeleteDeployment(name);
    if (!deleteRes.ok) {
        return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings };
    }

    return cleanupDestroyedServer(name, operation);
}

async function cleanupDestroyedServer(name: string, operation: string): Promise<DestroyServerResult> {
    const row = getServer(name);
    const warnings: string[] = [];
    // Persist the checkpoint before local work so failed cleanup can resume without RCON.
    // Legacy operations already recorded this same stage after Komodo deletion.
    deploymentStage(operation, REMOVING_SERVER);
    // Host directory. Fall back to derived path if host_dir wasn't set on
    // legacy rows.
    const hostDir = row?.host_dir ?? serverDir(name);
    try {
        await rm(hostDir, { recursive: true, force: true });
    } catch (err) {
        warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
    }

    // The registry trigger preserves history and marks its old server identity.
    // Live inventory, rollback snapshots, and restart slots still cascade away.
    try {
        db().prepare('DELETE FROM servers WHERE name = ?').run(name);
    } catch (err) {
        return {
            ok: false,
            kind: 'db',
            message: `failed to remove DB rows for ${name}: ${(err as Error).message}`,
            warnings
        };
    }

    const unfinished = db().prepare("SELECT id, action FROM server_deployments WHERE server = ? AND state = 'awaiting-health'").all(name) as { id: string; action: string }[];
    for (const { id, action } of unfinished) finishDeployment(id, { ok: false, message: action === 'create'
        ? 'Server destroyed before creation health checks completed' : 'Server destroyed before operation verification completed' });
    return { ok: true, warnings };
}
interface CreationJob {
    deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
    submitted_at?: string | null;
}

async function provisionCreation(id: string): Promise<CreateServerResult> {
    const job = db().prepare('SELECT j.*, d.triggered_by FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE j.deployment_id = ?').get(id) as CreationJob;
    creating = true;
    db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(id);
    try {
        const mapEntry = zombieMapBySlug(gameOfServer(job.server), job.map_slug);
        const plutoKey = getServerKey(job.server);
        if (!mapEntry || !plutoKey) {
            const result = { ok: false as const, kind: 'bad-request' as const, message: 'Saved server key or selected map is no longer available' };
            finishDeployment(id, result);
            return result;
        }
        const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by },
            { ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
        if (!result.ok) finishDeployment(id, result);
        else {
            db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
            deploymentStage(id, 'Waiting for Komodo health');
        }
        return result;
    } catch (err) {
        const server = getServer(job.server);
        if (server?.game_port === job.port) {
            // A lost response may have happened after Komodo started the server.
            // Keep its slot and observe readiness instead of starting another boot.
            db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
            recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message);
            deploymentStage(id, 'Verifying existing server after interrupted startup');
        } else finishDeployment(id, { ok: false, message: (err as Error).message });
        throw err;
    } finally { creating = false; }
}

type CreationOutcomes = Map<string, CreateServerResult | Error>;
let creationTick: Promise<CreationOutcomes> | null = null;
function creationPass(): Promise<CreationOutcomes> {
    if (creationTick) return creationTick;
    creationTick = runCreations().finally(() => { creationTick = null; });
    return creationTick;
}

export async function processCreations(): Promise<void> {
    await creationPass();
}

/** One host-wide FIFO, shared by collections and individual launches. */
export function creationQueue() {
    type Entry = { id: string; server: string; state: string; stage: string };
    const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments
        WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
        ORDER BY started_at, rowid`).all() as Entry[];
    return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
}

function creationBlocker(id: string): string | null {
    const dependency = db().prepare(`SELECT d.server, d.state FROM server_creation_dependencies dep
        JOIN server_deployments d ON d.id = dep.removal_id
        WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { server: string; state: string } | undefined;
    if (dependency) return `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`;
    const occupied = db().prepare(`SELECT s.name FROM servers s JOIN server_creation_jobs j ON s.game_port = j.port
        WHERE j.deployment_id = ? LIMIT 1`).get(id) as { name: string } | undefined;
    return occupied ? `Waiting for port to be released by ${occupied.name}` : null;
}

async function runCreations(): Promise<CreationOutcomes> {
    const outcomes: CreationOutcomes = new Map();
    if (creating || destroying.size > 0) return outcomes;
    // Restart recovery is observation-only: never repeat external creation or
    // deployment. A known interrupted startup must retain its readiness gate.
    const interrupted = db().prepare("SELECT id FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
    for (const { id } of interrupted) {
        if (resumeVerification(id)) continue;
        // A queued removal does not make the interrupted container disappear.
        // Keep its slot until removal finishes, without resuming normal work or
        // confusing it with a newer creation of the same server identity.
        const removing = db().prepare(`SELECT 1 FROM server_deployments d
            JOIN server_creation_jobs j ON j.deployment_id = d.id
            JOIN servers s ON s.name = j.server AND s.game_port = j.port
            WHERE d.id = ? AND EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server
                AND newer.rowid > d.rowid AND newer.action = 'destroy' AND newer.state IN ('waiting', 'running'))
            AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server
                AND newer.rowid > d.rowid AND newer.action = 'create')`).get(id);
        if (removing) {
            db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
            deploymentStage(id, 'Waiting for requested server removal after interrupted startup');
        }
    }

    // Check only previously started servers. Newly launched servers keep their
    // slot until a later pass, even if a health probe would answer immediately.
    const checking = db().prepare(`SELECT j.*, COALESCE(d.submitted_at, d.started_at) AS submitted_at FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
        WHERE d.server_deleted_at IS NULL AND d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[];
    for (const job of checking) {
        const id = job.deployment_id;
        if (getServer(job.server)?.game_port !== job.port) {
            finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' });
            continue;
        }
        if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
        try {
            const komodo = await komodoInspectHealthy(job.server);
            if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
            const status = await probeServerStatus(job.server, job.submitted_at);
            if (!status.ok) { deploymentStage(id, 'Waiting for RCON or fresh telemetry health: ' + status.reason); continue; }
            const actualMap = status.map;
            if (actualMap !== job.expected_map) {
                deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', ' + (status.source === 'rcon' ? 'RCON' : 'telemetry') + ' reports ' + (actualMap ?? 'no map'));
                continue;
            }
            deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap);
            finishDeployment(id, { ok: true });
        } catch (err) {
            deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message);
        }
    }

    // Snapshot the budget once. Fast success/failure must not turn a single
    // worker pass into an unbounded stream of expensive container startups.
    const queue = creationQueue();
    const available = Math.max(0, fleetSettings().createConcurrency - queue.active.length);
    const eligible = queue.queued.filter(({ id }) => {
        const blocker = creationBlocker(id);
        if (blocker) deploymentStage(id, blocker);
        return !blocker;
    });
    for (const { id } of eligible.slice(0, available)) {
        if (creating || destroying.size > 0) break;
        // API calls can cancel a selected job or resume an older startup while
        // a previous provision awaits Komodo. Recheck without expanding this pass.
        if (creationQueue().active.length >= fleetSettings().createConcurrency) break;
        if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
        const blocker = creationBlocker(id);
        if (blocker) {
            deploymentStage(id, blocker);
            continue;
        }
        try { outcomes.set(id, await provisionCreation(id)); }
        catch (err) {
            const failure = err instanceof Error ? err : new Error(String(err));
            outcomes.set(id, failure);
            console.error('[creation] provisioning failed:', failure);
        }
    }
    return outcomes;
}