import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js'; import { beginDeployment, deploymentStage, finishDeployment } from './deployments.js'; import { mkdir, chmod, rm, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js'; import { db } from '../db/index.js'; import { getServerKey, saveServerKey } from './server-keys.js'; import { getServer } from './servers.js'; import { zombieMapBySlug, type GameMode, type ZombieMapEntry, type ZombieGame } from './map-aliases.js'; import { komodoCreateDeployment, komodoDeploy, komodoDestroyDeployment, komodoDeleteDeployment, komodoInspectHealthy, komodoEnsureTag, komodoUpdateTagsOnResource, type CreateDeploymentSpec, type DeploymentEnvVar } from '../integrations/komodo.js'; import { kick } from '../pipeline/worker.js'; import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js'; import { recordScript } from './inventory.js'; import { sha256Buffer } from './files.js'; import { promotedArtifacts } from './promoted-artifacts.js'; export type CreateServerFailure = | 'bad-request' | 'unconfigured' // Komodo creds / target server missing | 'unknown-map' // slug isn't in the catalog | 'no-slot' // all lobby slots for this map are taken | 'no-port' // port range exhausted | 'fs' // couldn't create host dir | 'komodo' // Komodo CreateDeployment / Deploy rejected | 'db'; // shouldn't happen, but wraps INSERT failures export type CreateServerResult = | { ok: true; name: string; port: number; deploymentId: string } | { ok: false; kind: CreateServerFailure; message: string }; export type DestroyServerResult = | { ok: true; warnings: string[] } | { ok: false; kind: 'not-found' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] }; export interface CreateServerInput { game?: ZombieGame; // omitted by older clients: T5 gameMode: GameMode; // 'zombie' — only value supported today mapSlug: string; // 'cosmodrome' plutoKey?: string; // saved for this server ID when supplied expectedName?: string; // optional preview guard; never selects a lobby createdBy?: string; // 'ui' by default } /** * Find the lowest lobby number `n` such that `t5___` isn't in * the servers table. Caps at 20 to stay inside the port range — with 20 total * ports across the whole fleet, no one map can hold more than that. */ function nextLobbyNumber(baseName: string): number | null { const rows = db() .prepare(`SELECT name FROM servers WHERE name LIKE ? UNION SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')`) .all(`${baseName}_%`) as Array<{ name: string }>; const taken = new Set(); const prefix = `${baseName}_`; for (const r of rows) { if (!r.name.startsWith(prefix)) continue; const tail = r.name.slice(prefix.length); if (/^\d+$/.test(tail)) taken.add(Number.parseInt(tail, 10)); } for (let n = 1; n <= 20; n++) { if (!taken.has(n)) return n; } return null; } /** * Lowest unused port in the configured range. Reads every server's game_port * from the DB (which EC controls end-to-end under the new naming scheme, so * the DB is authoritative — no need to ask Komodo). */ function nextAvailablePort(): number | null { const rows = db() .prepare(`SELECT game_port FROM servers WHERE game_port IS NOT NULL UNION SELECT j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')`) .all() as Array<{ game_port: number }>; const taken = new Set(rows.map((r) => r.game_port)); for (let p = config.portRange.min; p <= config.portRange.max; p++) { if (!taken.has(p)) return p; } return null; } function buildEnv(args: { game: ZombieGame; name: string; port: number; plutoKey: string; }): DeploymentEnvVar[] { return [ { variable: 'PLUTAINER_SERVER_NAME', value: args.name }, { variable: 'PLUTAINER_PORT', value: String(args.port) }, { variable: 'PLUTAINER_GAME', value: args.game === 't6' ? 't6zm' : 't5sp' }, { variable: 'PLUTAINER_CONFIG_FILE', value: configFileForGame(args.game) }, // Plutainer seeds/loads the main config first. Load our generated map // config afterward, then rotate. A separate exec avoids quoted values // in EXTRA_ARGS, which Plutainer splits on whitespace. { variable: 'PLUTAINER_MAP_ROTATE', value: 'false' }, // Give telemetry scripts the registry name before the first map starts. { variable: 'PLUTAINER_EXTRA_ARGS', value: `+exec ${MAP_CONFIG_FILE} +set tws_server_id ${args.name} +map_rotate` }, { variable: 'PLUTO_SERVER_KEY', value: args.plutoKey }, { variable: 'PLUTAINER_RCON_PASSWORD', value: config.rcon.password } ]; } function buildSpec(args: { game: ZombieGame; name: string; port: number; hostDir: string; plutoKey: string; }): CreateDeploymentSpec { return { name: args.name, server: config.komodo.targetServer, image: config.komodo.defaultImage, network: config.komodo.defaultNetwork, // Container listens on PLUTAINER_PORT (== args.port), so host and // container ports match — no NAT inside the bridge network. ports: [ `${args.port}:${args.port}/udp`, `${args.port}:${args.port}/tcp` ], volumes: [ `${args.hostDir}:/home/plutainer/app`, `${args.game === 't6' ? config.komodo.t6GameFilesVolume : config.komodo.gameFilesVolume}:/home/plutainer/gamefiles:ro` ], environment: buildEnv(args) }; } let creating = false; interface PreparedCreation { ok: true; name: string; port: number; plutoKey: string; mapEntry: ZombieMapEntry; } export function queueCreateServer(input: CreateServerInput): CreateServerResult { return db().transaction((): CreateServerResult => { const prepared = prepareCreation(input); if (!prepared.ok) return prepared; const id = beginDeployment(prepared.name, 'create', input.createdBy); db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id); db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)') .run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical); deploymentStage(id, 'Queued for server creation'); return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id }; })(); } // Await provisioning for internal callers; completion still requires worker health verification. export async function createServer(input: CreateServerInput): Promise { if (creating || destroying.size > 0) return { ok: false, kind: 'no-slot', message: 'Another server operation is in progress. Please retry shortly.' }; const queued = queueCreateServer(input); if (!queued.ok) return queued; return await provisionCreation(queued.deploymentId); } function prepareCreation(input: CreateServerInput): PreparedCreation | Extract { const game = input.game ?? 't5'; if (game !== 't5' && game !== 't6') { return { ok: false, kind: 'bad-request', message: 'Supported games are t5 and t6' }; } if (input.gameMode !== 'zombie') { return { ok: false, kind: 'unknown-map', message: `unsupported game mode: ${input.gameMode}` }; } if (!config.komodo.address || !config.komodo.apiKey || !config.komodo.apiSecret) { return { ok: false, kind: 'unconfigured', message: 'KOMODO_ADDRESS / KOMODO_API_KEY / KOMODO_API_SECRET must be set' }; } if (!config.komodo.targetServer) { return { ok: false, kind: 'unconfigured', message: 'KOMODO_TARGET_SERVER must name the Komodo host that runs new deployments' }; } const mapEntry = zombieMapBySlug(game, input.mapSlug); if (!mapEntry) { return { ok: false, kind: 'unknown-map', message: `no ${game} zombie map with slug '${input.mapSlug}'` }; } const baseName = `${game}_${input.gameMode}_${mapEntry.slug}`; const lobby = nextLobbyNumber(baseName); if (lobby === null) { return { ok: false, kind: 'no-slot', message: `all lobby slots (1..20) for ${baseName} are taken` }; } const name = `${baseName}_${lobby}`; if (input.expectedName !== undefined && input.expectedName !== name) { return { ok: false, kind: 'no-slot', message: 'Available lobby changed. Review the updated server ID and key, then deploy again.' }; } const plutoKey = input.plutoKey?.trim() || getServerKey(name); if (!plutoKey) return { ok: false, kind: 'bad-request', message: `No saved key for ${name}. Enter a Plutonium server key.` }; const port = nextAvailablePort(); if (port === null) { return { ok: false, kind: 'no-port', message: `no free port in ${config.portRange.min}..${config.portRange.max} — destroy an existing server first` }; } // Save before external operations so a failed deployment can also be retried. try { saveServerKey(name, plutoKey); } catch (err) { return { ok: false, kind: 'bad-request', message: (err as Error).message }; } return { ok: true, name, port, plutoKey, mapEntry }; } async function createServerUnlocked(input: CreateServerInput, prepared: PreparedCreation, operation: string): Promise { const { name, port, plutoKey, mapEntry } = prepared; const game = gameOfServer(name); const configFile = configFileForGame(game); deploymentStage(operation, 'Preparing server files'); const hostDir = serverDir(name); const initialConfig = getPromotedServerConfig(game, mapEntry.canonical, mapEntry.slug); const mapConfig = getServerMapConfig(game, mapEntry.canonical, mapEntry.slug)!; const initialArtifacts = promotedArtifacts(game, mapEntry.canonical); // Step 1: filesystem. mkdir mode is masked by umask, so chmod explicitly // after — plutainer expects to be able to write to /home/plutainer/app. try { await mkdir(hostDir, { recursive: true, mode: 0o777 }); await chmod(hostDir, 0o777); for (const { kind, script, version } of initialArtifacts) { const dest = destOfKind(name, kind); const target = join(dest, script.target_rel_path); await mkdir(dirname(target), { recursive: true, mode: 0o777 }); // Make every newly created ancestor writable by Plutainer, even // when the manager's umask removed write permissions at mkdir. for (let dir = dirname(target); dir !== join(hostDir) && dir !== dirname(dir); dir = dirname(dir)) { await chmod(dir, 0o777); } await writeFile(target, version.content); await chmod(target, 0o666); } const mapConfigPath = join(destOfKind(name, 'config'), MAP_CONFIG_FILE); await mkdir(dirname(mapConfigPath), { recursive: true, mode: 0o777 }); await chmod(dirname(mapConfigPath), 0o777); await writeFile(mapConfigPath, mapConfig); await chmod(mapConfigPath, 0o666); if (initialConfig) { const configPath = join(destOfKind(name, 'config'), configFile); await mkdir(dirname(configPath), { recursive: true, mode: 0o777 }); await chmod(dirname(configPath), 0o777); await writeFile(configPath, initialConfig); await chmod(configPath, 0o666); } } catch (err) { return { ok: false, kind: 'fs', message: `failed to create ${hostDir}: ${(err as Error).message}` }; } // Step 2: Komodo CreateDeployment. deploymentStage(operation, 'Creating Komodo deployment'); const spec = buildSpec({ game, name, port, hostDir, plutoKey }); const createRes = await komodoCreateDeployment(spec); if (!createRes.ok) { await rm(hostDir, { recursive: true, force: true }).catch(() => {}); return { ok: false, kind: 'komodo', message: `CreateDeployment: ${createRes.message}` }; } // Step 3: DB row — insert *before* execute/Deploy so the sweep sees the // server immediately even if Deploy hangs. try { db().transaction(() => { db().prepare( `INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, host_dir, config_file, game_id, image, game_port, created_by, container_state) VALUES (?, 'deployment', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'starting')` ).run( name, name, game, mapEntry.canonical, input.gameMode, mapEntry.slug, hostDir, configFile, game === 't6' ? 't6zm' : 't5sp', config.komodo.defaultImage, port, input.createdBy ?? 'ui' ); if (initialConfig) { recordScript(name, configFile, null, sha256Buffer(initialConfig), initialConfig.byteLength, 'create'); } recordScript(name, MAP_CONFIG_FILE, null, sha256Buffer(mapConfig), mapConfig.byteLength, 'create-map'); for (const { script, version } of initialArtifacts) { recordScript(name, script.target_rel_path, version.id, version.sha256, version.size, 'create'); } })(); } catch (err) { // Roll back Komodo + FS if the DB insert fails (very unlikely — most // often a race where the same name got inserted twice). await komodoDestroyDeployment(name).catch(() => {}); await komodoDeleteDeployment(name).catch(() => {}); await rm(hostDir, { recursive: true, force: true }).catch(() => {}); return { ok: false, kind: 'db', message: `failed to insert server row: ${(err as Error).message}` }; } // Step 4: apply Komodo tags so operators can filter the deployment list // by game/map. Best-effort — a tag failure doesn't roll back the server; // the deployment is fully functional without cosmetic tags. Fires in // parallel to keep the create latency down. void applyDeploymentTags(name, game, mapEntry.canonical); // Step 5: execute/Deploy — actually start the container. deploymentStage(operation, 'Starting container'); const deployRes = await komodoDeploy(name); if (!deployRes.ok) { // Container never started — undo everything so retrying is clean. await komodoDeleteDeployment(name).catch(() => {}); db().prepare('DELETE FROM servers WHERE name = ?').run(name); await rm(hostDir, { recursive: true, force: true }).catch(() => {}); return { ok: false, kind: 'komodo', message: `Deploy: ${deployRes.message}` }; } // Step 6: kick the sweep so RCON status polling for this server starts // now instead of on the next 15s tick. void kick().catch((err) => console.error('[createServer] post-create kick failed:', err)); return { ok: true, name, port, deploymentId: operation }; } /** * Ensure `game` and `map` tags exist in Komodo, then attach both to the given * deployment. Purely cosmetic — any failure here logs and returns; the caller * treats it as fire-and-forget. */ async function applyDeploymentTags(deployment: string, gameTagName: string, mapTagName: string): Promise { try { const [gameTag, mapTag] = await Promise.all([ komodoEnsureTag(gameTagName), komodoEnsureTag(mapTagName) ]); const tagIds: string[] = []; if (gameTag.ok) tagIds.push(gameTag.id); else console.warn(`[createServer] tag "${gameTagName}": ${gameTag.message}`); if (mapTag.ok) tagIds.push(mapTag.id); else console.warn(`[createServer] tag "${mapTagName}": ${mapTag.message}`); if (tagIds.length === 0) return; const res = await komodoUpdateTagsOnResource('Deployment', deployment, tagIds); if (!res.ok) { console.warn(`[createServer] UpdateResourceMeta (tags) for ${deployment}: ${res.message}`); } } catch (err) { console.warn(`[createServer] tag application for ${deployment} threw:`, err); } } const destroying = new Set(); // Persist the request before returning so navigation and client disconnects cannot lose it. export function destroyServer(name: string): DestroyServerResult & { deploymentId?: string } { const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined; if (existing) return { ok: true, warnings: [], deploymentId: existing.id }; if (!getServer(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] }; const operation = db().transaction(() => { const id = beginDeployment(name, 'destroy'); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); deploymentStage(id, 'Waiting to verify the server is empty'); return id; })(); return { ok: true, warnings: [], deploymentId: operation }; } let destructionTick: Promise | null = null; export function processDestructions(): Promise { if (destructionTick) return destructionTick; destructionTick = runDestructions().finally(() => { destructionTick = null; }); return destructionTick; } async function runDestructions(): Promise { const queued = db().prepare("SELECT id, server FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string }[]; for (const operation of queued) { if (creating || destroying.has(operation.server)) continue; destroying.add(operation.server); try { if (!getServer(operation.server)) { finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' }); continue; } // A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness. const players = await rconGetPlayerCount(operation.server); if (players !== 0) { deploymentStage(operation.id, players === null ? 'Waiting for player count: server could not be reached' : 'Waiting for server to empty: ' + players + ' player(s) online'); continue; } deploymentStage(operation.id, 'Server is empty (0 players verified)'); db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(operation.id); const result = await destroyServerUnlocked(operation.server, operation.id); finishDeployment(operation.id, result); } catch (err) { const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string }; if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message); else finishDeployment(operation.id, { ok: false, message: (err as Error).message }); } finally { destroying.delete(operation.server); } } } async function destroyServerUnlocked(name: string, operation: string): Promise { const row = getServer(name); if (!row) { return { ok: false, kind: 'not-found', message: `no server named '${name}'`, warnings: [] }; } const warnings: string[] = []; // 1: stop + remove the container. execute/DestroyDeployment queues the // teardown; the response comes back before the container is actually gone. deploymentStage(operation, 'Stopping and removing container'); const destroyRes = await komodoDestroyDeployment(name); if (!destroyRes.ok) { return { ok: false, kind: 'komodo', message: 'DestroyDeployment failed: ' + destroyRes.message, warnings }; } // 2: wait for the container to actually disappear. Komodo refuses to // DeleteDeployment while a container still exists for it, so poll the // health probe until it stops reporting a running state. `ok:false` here // is the *success* signal — it means the container is no longer inspectable. const pollDeadline = Date.now() + 30_000; while (Date.now() < pollDeadline) { const health = await komodoInspectHealthy(name); if (!health.ok) break; await new Promise((resolve) => setTimeout(resolve, 1500)); } // 3: remove the deployment entity itself. deploymentStage(operation, 'Deleting Komodo deployment'); const deleteRes = await komodoDeleteDeployment(name); if (!deleteRes.ok) { return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings }; } // 3: host directory. Fall back to derived path if host_dir wasn't set on // legacy rows. deploymentStage(operation, 'Removing server files and registry'); const hostDir = row.host_dir ?? serverDir(name); try { await rm(hostDir, { recursive: true, force: true }); } catch (err) { warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`); } // 4: DB row + cascading rows. ON DELETE CASCADE covers server_scripts, // beta_file_snapshots; the rest (deployment_servers, deployment_files, // beta_servers, beta_server_scripts) reference servers(name) without // cascade — clean them up explicitly. try { const tx = db().transaction((n: string) => { db().prepare('DELETE FROM deployment_servers WHERE server = ?').run(n); db().prepare('DELETE FROM deployment_files WHERE server = ?').run(n); db().prepare('DELETE FROM beta_servers WHERE server = ?').run(n); db().prepare('DELETE FROM beta_server_scripts WHERE server = ?').run(n); db().prepare('DELETE FROM servers WHERE name = ?').run(n); }); tx(name); } catch (err) { return { ok: false, kind: 'db', message: `failed to remove DB rows for ${name}: ${(err as Error).message}`, warnings }; } const unfinished = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' AND state = 'awaiting-health'").all(name) as { id: string }[]; for (const { id } of unfinished) finishDeployment(id, { ok: false, message: 'Server destroyed before creation health checks completed' }); return { ok: true, warnings }; } interface CreationJob { deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string; } async function provisionCreation(id: string): Promise { const job = db().prepare('SELECT j.*, d.triggered_by FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE j.deployment_id = ?').get(id) as CreationJob; creating = true; db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(id); try { const mapEntry = zombieMapBySlug(gameOfServer(job.server), job.map_slug); const plutoKey = getServerKey(job.server); if (!mapEntry || !plutoKey) { const result = { ok: false as const, kind: 'bad-request' as const, message: 'Saved server key or selected map is no longer available' }; finishDeployment(id, result); return result; } const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by }, { ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id); if (!result.ok) finishDeployment(id, result); else { db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id); deploymentStage(id, 'Waiting for Komodo health'); } return result; } catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); throw err; } finally { creating = false; } } let creationTick: Promise | null = null; export function processCreations(): Promise { if (creationTick) return creationTick; creationTick = runCreations().finally(() => { creationTick = null; }); return creationTick; } async function runCreations(): Promise { if (creating || destroying.size > 0) return; const queued = db().prepare("SELECT id FROM server_deployments WHERE action = 'create' AND state = 'queued' ORDER BY started_at").all() as { id: string }[]; for (const { id } of queued) { if (creating || destroying.size > 0) break; await provisionCreation(id).catch(err => console.error('[creation] provisioning failed:', err)); } const checking = db().prepare("SELECT j.* FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state = 'awaiting-health'").all() as CreationJob[]; for (const job of checking) { const id = job.deployment_id; if (!getServer(job.server)) { finishDeployment(id, { ok: false, message: 'Server was removed before creation health checks completed' }); continue; } if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue; try { const komodo = await komodoInspectHealthy(job.server); if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; } const rcon = await rconStatusResult(job.server); if (!rcon.ok) { deploymentStage(id, 'Waiting for RCON health: ' + rcon.reason); continue; } const actualMap = rcon.status.map; if (actualMap !== job.expected_map) { deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', RCON reports ' + (actualMap ?? 'no map')); continue; } deploymentStage(id, 'Verified Komodo healthy, RCON healthy, and map ' + actualMap); finishDeployment(id, { ok: true }); } catch (err) { deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message); } } }