1032 lines
65 KiB
TypeScript
1032 lines
65 KiB
TypeScript
import { listHosts } from '../hosts/index.js';
|
|
import { fleetSettings } from '../settings.js';
|
|
import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence.js';
|
|
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
|
|
import { randomUUID } from 'node:crypto';
|
|
import type { LaunchPreview } from '$lib/deployment-plan.js';
|
|
import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
|
|
import { mkdir, chmod, rm, writeFile } from 'node:fs/promises';
|
|
import { dirname, join } from 'node:path';
|
|
import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js';
|
|
import { db } from '../db/index.js';
|
|
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
|
|
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
|
import { zombieMapBySlug, type GameMode, type ZombieMapEntry, type ZombieGame } from '../fleet/map-aliases.js';
|
|
import {
|
|
komodoCreateDeployment,
|
|
komodoDeploy,
|
|
komodoDestroyDeployment,
|
|
komodoDeleteDeployment,
|
|
komodoInspectHealthy,
|
|
komodoEnsureTag,
|
|
komodoUpdateTagsOnResource,
|
|
type CreateDeploymentSpec,
|
|
type DeploymentEnvVar
|
|
} from '../integrations/komodo.js';
|
|
import { MAP_CONFIG_FILE } from '../fleet/server-config.js';
|
|
import { recordScript } from '../fleet/inventory.js';
|
|
import { getVersion } from '../fleet/scripts.js';
|
|
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
|
import { recordProgress, type ProgressUpdate } from './progress.js';
|
|
import { checkRelease, freshRelease, probeVersion, release } from '../integrations/plutonium.js';
|
|
import { nativeInstance, nativeDeployment } from '../backends/native.js';
|
|
import { previewNativeCreation, queueNativeCreation, queueNativeRebuild, cancelQueuedNativeCreation } from './native.js';
|
|
|
|
export type CreateServerFailure =
|
|
| 'bad-request'
|
|
| 'conflict'
|
|
| 'unconfigured' // Komodo creds / target server missing
|
|
| 'unknown-map' // slug isn't in the catalog
|
|
| 'no-slot' // all lobby slots for this map are taken
|
|
| 'no-port' // port range exhausted
|
|
| 'fs' // couldn't create host dir
|
|
| 'komodo' // Komodo CreateDeployment / Deploy rejected
|
|
| 'db'; // shouldn't happen, but wraps INSERT failures
|
|
|
|
export type CreateServerResult =
|
|
| { ok: true; name: string; port: number | null; deploymentId: string }
|
|
| { ok: false; kind: CreateServerFailure; message: string };
|
|
|
|
export type DestroyServerResult =
|
|
| { ok: true; warnings: string[] }
|
|
| { ok: false; kind: 'not-found' | 'conflict' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] };
|
|
|
|
export interface CreateServerInput {
|
|
hostId?: string;
|
|
libraryId?: string;
|
|
image?: string;
|
|
bindAddress?: string;
|
|
publicAddress?: string;
|
|
publicPort?: number;
|
|
game?: ZombieGame; // omitted by older clients: T5
|
|
gameMode: GameMode; // 'zombie' — only value supported today
|
|
mapSlug: string; // 'cosmodrome'
|
|
plutoKey?: string; // saved for this server ID when supplied
|
|
expectedName?: string; // optional preview guard; never selects a lobby
|
|
previewId?: string;
|
|
createdBy?: string; // 'ui' by default
|
|
}
|
|
|
|
/**
|
|
* Find the lowest lobby number `n` such that `t5_<mode>_<slug>_<n>` isn't in
|
|
* the servers table. Caps at 20 to stay inside the port range — with 20 total
|
|
* ports across the whole fleet, no one map can hold more than that.
|
|
*/
|
|
export function nextLobbyNumber(baseName: string, additionalNames: string[] = []): number | null {
|
|
const rows = db()
|
|
.prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
|
|
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
|
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
|
|
.all(`${baseName}_%`) as Array<{ name: string }>;
|
|
const taken = new Set<number>();
|
|
const prefix = `${baseName}_`;
|
|
for (const r of [...rows, ...additionalNames.map(name => ({ name }))]) {
|
|
if (!r.name.startsWith(prefix)) continue;
|
|
const tail = r.name.slice(prefix.length);
|
|
if (/^\d+$/.test(tail)) taken.add(Number.parseInt(tail, 10));
|
|
}
|
|
for (let n = 1; n <= 20; n++) {
|
|
if (!taken.has(n)) return n;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Lowest unused port in the configured range. Reads every server's game_port
|
|
* from the DB (which EC controls end-to-end under the new naming scheme, so
|
|
* the DB is authoritative — no need to ask Komodo).
|
|
*/
|
|
export function nextAvailablePort(additionalPorts: number[] = [], releasingServers: string[] = []): number | null {
|
|
const rows = db()
|
|
.prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
|
|
SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
|
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
|
|
.all() as Array<{ name: string; game_port: number }>;
|
|
const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
|
|
for (let p = config.portRange.min; p <= config.portRange.max; p++) {
|
|
if (!taken.has(p)) return p;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function buildEnv(args: {
|
|
game: ZombieGame;
|
|
name: string;
|
|
port: number;
|
|
plutoKey: string;
|
|
}): DeploymentEnvVar[] {
|
|
return [
|
|
{ variable: 'PLUTAINER_SERVER_NAME', value: args.name },
|
|
{ variable: 'PLUTAINER_PORT', value: String(args.port) },
|
|
{ variable: 'PLUTAINER_GAME', value: args.game === 't6' ? 't6zm' : 't5sp' },
|
|
{ variable: 'PLUTAINER_CONFIG_FILE', value: configFileForGame(args.game) },
|
|
// Plutainer seeds/loads the main config first. Load our generated map
|
|
// config afterward, then rotate. A separate exec avoids quoted values
|
|
// in EXTRA_ARGS, which Plutainer splits on whitespace.
|
|
{ variable: 'PLUTAINER_MAP_ROTATE', value: 'false' },
|
|
// Give telemetry scripts the registry name before the first map starts.
|
|
{ variable: 'PLUTAINER_EXTRA_ARGS', value: `+exec ${MAP_CONFIG_FILE} +set tws_server_id ${args.name} +map_rotate` },
|
|
{ variable: 'PLUTO_SERVER_KEY', value: args.plutoKey },
|
|
{ variable: 'PLUTAINER_RCON_PASSWORD', value: config.rcon.password }
|
|
];
|
|
}
|
|
|
|
function buildSpec(args: {
|
|
game: ZombieGame;
|
|
name: string;
|
|
port: number;
|
|
hostDir: string;
|
|
plutoKey: string;
|
|
recipe: LaunchRecipe;
|
|
}): CreateDeploymentSpec {
|
|
return {
|
|
name: args.name,
|
|
server: config.komodo.targetServer,
|
|
image: args.recipe.image,
|
|
network: config.komodo.defaultNetwork,
|
|
// Container listens on PLUTAINER_PORT (== args.port), so host and
|
|
// container ports match — no NAT inside the bridge network.
|
|
ports: [
|
|
`${args.port}:${args.port}/udp`,
|
|
`${args.port}:${args.port}/tcp`
|
|
],
|
|
volumes: [
|
|
`${args.hostDir}:/home/plutainer/app`,
|
|
`${args.game === 't6' ? config.komodo.t6GameFilesVolume : config.komodo.gameFilesVolume}:/home/plutainer/gamefiles:ro`
|
|
],
|
|
environment: buildEnv(args)
|
|
};
|
|
}
|
|
|
|
let creating = false;
|
|
async function observeCommand<T>(operation: string, command: () => Promise<T>): Promise<T> {
|
|
try { return await command(); }
|
|
catch (err) {
|
|
recordProgress(operation, { command: { state: 'uncertain', message: 'No confirmed command result was received: ' + (err as Error).message } });
|
|
throw err;
|
|
}
|
|
}
|
|
interface PreparedCreation {
|
|
ok: true; name: string; port: number; plutoKey: string; mapEntry: ZombieMapEntry;
|
|
}
|
|
|
|
interface PreviewRow {
|
|
id: string; expires_at: string; server: string | null; port: number | null;
|
|
recipe: string; blueprint_name: string | null; deployment_id: string | null;
|
|
}
|
|
|
|
/** Read-only checks; a preview never reserves a slot or writes credentials. */
|
|
export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown; hostId?: unknown; libraryId?: unknown }): LaunchPreview {
|
|
if (input.hostId !== undefined || !config.komodo.address || listHosts().length > 0) return previewNativeCreation(input);
|
|
if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.');
|
|
let recipe: LaunchRecipePlan;
|
|
let blueprintName: string | null = null;
|
|
if (input.blueprintId !== undefined) {
|
|
if (typeof input.blueprintId !== 'string') throw new Error('Invalid blueprint.');
|
|
({ recipe, blueprintName } = blueprintRecipe(input.blueprintId));
|
|
} else if (input.deploymentId !== undefined) {
|
|
if (typeof input.deploymentId !== 'string') throw new Error('Invalid deployment.');
|
|
const source = recipeForDeployment(input.deploymentId);
|
|
if (!source) throw new Error('This deployment has no recorded launch recipe.');
|
|
blueprintName = source.blueprintName;
|
|
// History describes the previous launch; only its map selection is reusable.
|
|
recipe = captureRecipePlan(source.recipe.game, source.recipe.mapSlug);
|
|
} else {
|
|
if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.');
|
|
recipe = captureRecipePlan(input.game, input.mapSlug);
|
|
}
|
|
const lobby = nextLobbyNumber(`${recipe.game}_zombie_${recipe.mapSlug}`);
|
|
const name = lobby === null ? null : `${recipe.game}_zombie_${recipe.mapSlug}_${lobby}`;
|
|
const port = nextAvailablePort();
|
|
const configured = !!(config.komodo.address && config.komodo.apiKey && config.komodo.apiSecret && config.komodo.targetServer);
|
|
const checks: LaunchPreview['checks'] = [
|
|
{ label: 'Komodo settings', state: configured ? 'pass' : 'blocked', detail: configured ? 'Connection settings and target host are configured.' : 'Configure Komodo credentials and a target host before launching.' },
|
|
{ label: 'Health verification', state: config.rcon.password ? 'pass' : 'blocked', detail: config.rcon.password ? 'RCON is configured. Container, RCON, and map checks run after launch.' : 'Configure RCON_PASSWORD so the new server can be verified.' },
|
|
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All 20 lobbies for this map are occupied or reserved.' },
|
|
{ label: 'Game port', state: port === null ? 'blocked' : 'pass', detail: port === null ? 'No free game port. Free capacity before launching.' : `${port} / UDP and TCP. Availability is checked again at launch.` },
|
|
{ label: 'Promoted configuration', state: 'pass', detail: recipe.configSource },
|
|
{ label: 'Game files and host readiness', state: 'info', detail: 'The host must have the selected game files. This preview does not probe volumes, images, credentials, or network access.' }
|
|
];
|
|
const id = randomUUID();
|
|
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
|
db().prepare('DELETE FROM server_launch_previews WHERE expires_at < ? AND deployment_id IS NULL').run(new Date().toISOString());
|
|
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
|
|
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
|
|
return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName,
|
|
checks, ready: checks.every(check => check.state !== 'blocked') };
|
|
}
|
|
|
|
// releasingServers is internal to atomic collection queueing; HTTP callers cannot set it.
|
|
export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult {
|
|
const nativePreview = input.previewId ? db().prepare("SELECT 1 FROM server_launch_previews WHERE id=? AND json_extract(recipe,'$.native.hostId') IS NOT NULL").get(input.previewId) : null;
|
|
if (input.hostId !== undefined || nativePreview || !config.komodo.address || listHosts().length > 0) return queueNativeCreation(input);
|
|
return db().transaction((): CreateServerResult => {
|
|
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined;
|
|
if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' };
|
|
const recipe: LaunchRecipePlan | undefined = preview ? JSON.parse(preview.recipe) : undefined;
|
|
if (recipe && (recipe.game !== (input.game ?? 't5') || recipe.mapSlug !== input.mapSlug || input.gameMode !== 'zombie')) {
|
|
return { ok: false, kind: 'bad-request', message: 'The selection does not match the reviewed launch recipe.' };
|
|
}
|
|
// A repeated submit after a lost response returns the same operation.
|
|
if (preview?.deployment_id) return { ok: true, name: preview.server!, port: preview.port!, deploymentId: preview.deployment_id };
|
|
if (preview && preview.expires_at <= new Date().toISOString()) return { ok: false, kind: 'bad-request', message: 'Launch preview expired. Review a fresh preview.' };
|
|
if (preview && (!preview.server || !preview.port || preview.server !== nextName(input) || preview.port !== nextAvailablePort([], releasingServers))) {
|
|
return { ok: false, kind: 'no-slot', message: 'Available lobby or port changed. Refresh the preview and review the new server key.' };
|
|
}
|
|
if (recipe && !config.rcon.password) {
|
|
return { ok: false, kind: 'unconfigured', message: 'Health-check settings changed. Refresh the preview.' };
|
|
}
|
|
const prepared = prepareCreation(input, releasingServers);
|
|
if (!prepared.ok) return prepared;
|
|
// A preview records choices, not version pins. Promotions remain authoritative.
|
|
const currentRecipe = captureRecipePlan(input.game ?? 't5', prepared.mapEntry.slug);
|
|
const storedRecipe = recipeStorage(currentRecipe);
|
|
const id = beginDeployment(prepared.name, 'create', input.createdBy);
|
|
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary, blueprint_name) VALUES (?, ?, ?, ?)')
|
|
.run(id, storedRecipe.recipe, storedRecipe.recipeSummary, preview?.blueprint_name ?? null);
|
|
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id = ? WHERE id = ?').run(id, preview.id);
|
|
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
|
|
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
|
|
.run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical);
|
|
deploymentStage(id, 'Queued for server creation', { phase: 'startup-queue', recipeStatus: 'queued', observedAt: null,
|
|
blocker: { code: 'startup-queue', message: 'Waiting for startup admission and available capacity.', automatic: true } });
|
|
return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id };
|
|
})();
|
|
}
|
|
|
|
/** Reserve the same identity and port before removing any part of the instance. */
|
|
export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServerResult {
|
|
if (nativeInstance(name)) return queueNativeRebuild(name, triggeredBy);
|
|
const blocker = activeBetaRemovalBlocker(name);
|
|
if (blocker) return { ok: false, kind: 'conflict', message: blocker };
|
|
const server = getServer(name);
|
|
const game = gameOfServer(name);
|
|
const map = server?.map_slug && zombieMapBySlug(game, server.map_slug);
|
|
if (!server || !map || !server.game_port || server.komodo_type !== 'deployment' || server.komodo_name !== name
|
|
|| (server.host_dir && server.host_dir !== serverDir(name))) {
|
|
return { ok: false, kind: 'bad-request', message: 'Rebuild requires a managed server with its original map, directory, and port.' };
|
|
}
|
|
if (!config.komodo.targetServer || (!config.rcon.password && !triggeredBy.startsWith('watchdog:'))) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' };
|
|
if (!getServerKey(name)) return { ok: false, kind: 'bad-request', message: `Save a server key for ${name} in Keys before rebuilding.` };
|
|
// Validate that the replacement setup can be composed before destruction.
|
|
const recipe = captureRecipePlan(game, map.slug);
|
|
return db().transaction(() => {
|
|
const id = beginDeployment(name, 'rebuild', triggeredBy);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
|
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
|
|
.run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
|
|
db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
|
|
const storedRecipe = recipeStorage(recipe);
|
|
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary) VALUES (?, ?, ?)')
|
|
.run(id, storedRecipe.recipe, storedRecipe.recipeSummary);
|
|
deploymentStage(id, 'Waiting to rebuild the empty server', { phase: 'eligibility', recipeStatus: 'queued', observedAt: null,
|
|
blocker: { code: 'eligibility', message: 'Waiting for safe rebuild eligibility checks.', automatic: true } });
|
|
return { ok: true as const, name, port: server.game_port!, deploymentId: id };
|
|
})();
|
|
}
|
|
|
|
/** Caller has just verified emptiness and excluded conflicting work. */
|
|
export async function rebuildServer(id: string): Promise<void> {
|
|
const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
|
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
|
|
if (!job) return;
|
|
const blocker = activeBetaRemovalBlocker(job.server);
|
|
if (blocker) {
|
|
// Keep the cleanup checkpoint intact when resuming after remote deletion.
|
|
if (job.stage === REMOVING_SERVER) finishDeployment(id, { ok: false, message: blocker });
|
|
else deploymentStage(id, blocker, { phase: 'eligibility', blocker: betaRemovalEvidence(job.server, blocker), checks: [] });
|
|
return;
|
|
}
|
|
if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) {
|
|
// The cleanup checkpoint must survive capacity waits and app restarts.
|
|
const progress: ProgressUpdate = { phase: job.stage === REMOVING_SERVER ? 'cleanup' : 'eligibility',
|
|
blocker: { code: 'startup-capacity', message: 'Waiting for a server creation slot before rebuilding', automatic: true } };
|
|
if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding', progress);
|
|
else recordProgress(id, progress);
|
|
return;
|
|
}
|
|
// Recheck prerequisites while the original instance still exists.
|
|
if (!getServerKey(job.server)) { finishDeployment(id, { ok: false, message: 'Saved server key was removed; rebuild did not begin.' }); return; }
|
|
try { captureRecipePlan(gameOfServer(job.server), job.map_slug); }
|
|
catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); return; }
|
|
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
|
|
destroying.add(job.server);
|
|
try {
|
|
const result = job.stage === REMOVING_SERVER
|
|
? await cleanupDestroyedServer(job.server, id) : await destroyServerUnlocked(job.server, id);
|
|
if (!result.ok || result.warnings.length) {
|
|
finishDeployment(id, { ok: false, message: result.ok ? 'Old server files could not be completely removed.' : result.message, warnings: result.warnings });
|
|
return;
|
|
}
|
|
db().transaction(() => {
|
|
db().prepare('UPDATE server_rebuild_jobs SET removed_at = ? WHERE deployment_id = ?').run(new Date().toISOString(), id);
|
|
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
|
|
deploymentStage(id, 'Queued for server creation', { phase: 'startup-queue', checks: [],
|
|
blocker: { code: 'startup-queue', message: 'Old instance removed; waiting for fresh-instance startup admission.', automatic: true },
|
|
checkpoint: { key: 'cleanup', label: 'Old instance files and registry removed' } });
|
|
})();
|
|
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
|
|
finally { destroying.delete(job.server); }
|
|
}
|
|
|
|
function nextName(input: CreateServerInput): string | null {
|
|
const base = `${input.game ?? 't5'}_${input.gameMode}_${input.mapSlug}`;
|
|
const lobby = nextLobbyNumber(base);
|
|
return lobby === null ? null : `${base}_${lobby}`;
|
|
}
|
|
|
|
// Internal callers use the same durable queue and capacity limit as HTTP launches.
|
|
// Await this pass's provisioning when possible; a busy host leaves the job queued.
|
|
export async function createServer(input: CreateServerInput): Promise<CreateServerResult> {
|
|
const queued = queueCreateServer(input);
|
|
if (!queued.ok) return queued;
|
|
const existing = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(queued.deploymentId) as { state: string };
|
|
if (existing.state !== 'queued') return queued;
|
|
const outcomes = await creationPass();
|
|
const outcome = outcomes.get(queued.deploymentId);
|
|
if (outcome instanceof Error) throw outcome;
|
|
return outcome ?? queued;
|
|
}
|
|
|
|
function prepareCreation(input: CreateServerInput, releasingServers: string[] = []): PreparedCreation | Extract<CreateServerResult, { ok: false }> {
|
|
const game = input.game ?? 't5';
|
|
if (game !== 't5' && game !== 't6') {
|
|
return { ok: false, kind: 'bad-request', message: 'Supported games are t5 and t6' };
|
|
}
|
|
if (input.gameMode !== 'zombie') {
|
|
return { ok: false, kind: 'unknown-map', message: `unsupported game mode: ${input.gameMode}` };
|
|
}
|
|
if (!config.komodo.address || !config.komodo.apiKey || !config.komodo.apiSecret) {
|
|
return { ok: false, kind: 'unconfigured', message: 'KOMODO_ADDRESS / KOMODO_API_KEY / KOMODO_API_SECRET must be set' };
|
|
}
|
|
if (!config.komodo.targetServer) {
|
|
return { ok: false, kind: 'unconfigured', message: 'KOMODO_TARGET_SERVER must name the Komodo host that runs new deployments' };
|
|
}
|
|
|
|
const mapEntry = zombieMapBySlug(game, input.mapSlug);
|
|
if (!mapEntry) {
|
|
return { ok: false, kind: 'unknown-map', message: `no ${game} zombie map with slug '${input.mapSlug}'` };
|
|
}
|
|
|
|
const baseName = `${game}_${input.gameMode}_${mapEntry.slug}`;
|
|
const lobby = nextLobbyNumber(baseName);
|
|
if (lobby === null) {
|
|
return { ok: false, kind: 'no-slot', message: `all lobby slots (1..20) for ${baseName} are taken` };
|
|
}
|
|
const name = `${baseName}_${lobby}`;
|
|
if (input.expectedName !== undefined && input.expectedName !== name) {
|
|
return { ok: false, kind: 'no-slot', message: 'Available lobby changed. Review the updated server ID and key, then deploy again.' };
|
|
}
|
|
const plutoKey = input.plutoKey?.trim() || getServerKey(name);
|
|
if (!plutoKey) return { ok: false, kind: 'bad-request', message: `No saved key for ${name}. Enter a Plutonium server key.` };
|
|
|
|
const port = nextAvailablePort([], releasingServers);
|
|
if (port === null) {
|
|
return {
|
|
ok: false,
|
|
kind: 'no-port',
|
|
message: `no free port in ${config.portRange.min}..${config.portRange.max} — destroy an existing server first`
|
|
};
|
|
}
|
|
|
|
// Save before external operations so a failed deployment can also be retried.
|
|
try { saveServerKey(name, plutoKey); }
|
|
catch (err) { return { ok: false, kind: 'bad-request', message: (err as Error).message }; }
|
|
|
|
return { ok: true, name, port, plutoKey, mapEntry };
|
|
}
|
|
|
|
async function createServerUnlocked(input: CreateServerInput, prepared: PreparedCreation, operation: string): Promise<CreateServerResult> {
|
|
const { name, port, plutoKey, mapEntry } = prepared;
|
|
const game = gameOfServer(name);
|
|
// Resolve once more at provisioning so promotions made while queued also apply.
|
|
// Persist the exact attempted files for history, never as input to future launches.
|
|
const recipe = captureRecipe(game, mapEntry.slug);
|
|
const storedRecipe = recipeStorage(recipe);
|
|
db().transaction(() => {
|
|
db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary) VALUES (?, ?, ?)
|
|
ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe, recipe_summary = excluded.recipe_summary`)
|
|
.run(operation, storedRecipe.recipe, storedRecipe.recipeSummary);
|
|
deploymentStage(operation, 'Preparing server files', { phase: 'prepare', recipeStatus: 'attempted', blocker: null, checks: [],
|
|
checkpoint: { key: 'startup-queue', label: 'Admitted to startup' },
|
|
event: { kind: 'recipe', message: 'Promoted files selected for this creation attempt.', details: { capturedAt: recipe.capturedAt } } });
|
|
})();
|
|
const configFile = recipe.configFile;
|
|
const hostDir = serverDir(name);
|
|
|
|
// Step 1: filesystem. mkdir mode is masked by umask, so chmod explicitly
|
|
// after — plutainer expects to be able to write to /home/plutainer/app.
|
|
try {
|
|
await mkdir(hostDir, { recursive: true, mode: 0o777 });
|
|
await chmod(hostDir, 0o777);
|
|
for (const file of recipe.files) {
|
|
const dest = destOfKind(name, file.kind);
|
|
const target = join(dest, file.path);
|
|
await mkdir(dirname(target), { recursive: true, mode: 0o777 });
|
|
// Make every newly created ancestor writable by Plutainer, even
|
|
// when the manager's umask removed write permissions at mkdir.
|
|
for (let dir = dirname(target); dir !== join(hostDir) && dir !== dirname(dir); dir = dirname(dir)) {
|
|
await chmod(dir, 0o777);
|
|
}
|
|
await writeFile(target, Buffer.from(file.content, 'base64'));
|
|
await chmod(target, 0o666);
|
|
}
|
|
} catch (err) {
|
|
return { ok: false, kind: 'fs', message: `failed to create ${hostDir}: ${(err as Error).message}` };
|
|
}
|
|
|
|
// Step 2: Komodo CreateDeployment.
|
|
deploymentStage(operation, 'Creating Komodo deployment', { phase: 'create-deployment',
|
|
checkpoint: { key: 'prepare', label: 'Server files prepared' },
|
|
command: { state: 'attempted', message: 'Komodo deployment creation attempted.' } });
|
|
const spec = buildSpec({ game, name, port, hostDir, plutoKey, recipe });
|
|
const createRes = await observeCommand(operation, () => komodoCreateDeployment(spec));
|
|
if (!createRes.ok) {
|
|
recordProgress(operation, { command: { state: createRes.kind === 'unreachable' || (createRes.httpStatus !== undefined && createRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: createRes.message } });
|
|
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
|
|
return { ok: false, kind: 'komodo', message: `CreateDeployment: ${createRes.message}` };
|
|
}
|
|
recordProgress(operation, { command: { state: 'accepted', message: 'Komodo created the deployment.' },
|
|
checkpoint: { key: 'remote-deployment-created', phase: 'create-deployment', label: 'Komodo deployment created' } });
|
|
|
|
// Step 3: DB row — insert *before* execute/Deploy so the sweep sees the
|
|
// server immediately even if Deploy hangs.
|
|
try {
|
|
db().transaction(() => {
|
|
db().prepare(
|
|
`INSERT INTO servers
|
|
(name, komodo_type, komodo_name, game, map,
|
|
game_mode, map_slug, host_dir, config_file, game_id, image,
|
|
game_port, created_by, container_state)
|
|
VALUES
|
|
(?, 'deployment', ?, ?, ?,
|
|
?, ?, ?, ?, ?, ?,
|
|
?, ?, 'starting')`
|
|
).run(
|
|
name, name, game, mapEntry.canonical,
|
|
input.gameMode, mapEntry.slug, hostDir,
|
|
configFile, game === 't6' ? 't6zm' : 't5sp', recipe.image,
|
|
port, input.createdBy ?? 'ui'
|
|
);
|
|
for (const file of recipe.files) {
|
|
const versionId = file.versionId && getVersion(file.versionId) ? file.versionId : null;
|
|
recordScript(name, file.path, versionId, file.sha256, file.size, file.path === MAP_CONFIG_FILE ? 'create-map' : 'create');
|
|
}
|
|
})();
|
|
} catch (err) {
|
|
// Roll back Komodo + FS if the DB insert fails (very unlikely — most
|
|
// often a race where the same name got inserted twice).
|
|
await komodoDestroyDeployment(name).catch(() => {});
|
|
await komodoDeleteDeployment(name).catch(() => {});
|
|
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
|
|
return { ok: false, kind: 'db', message: `failed to insert server row: ${(err as Error).message}` };
|
|
}
|
|
|
|
// Step 4: apply Komodo tags so operators can filter the deployment list
|
|
// by game/map. Best-effort — a tag failure doesn't roll back the server;
|
|
// the deployment is fully functional without cosmetic tags. Fires in
|
|
// parallel to keep the create latency down.
|
|
void applyDeploymentTags(name, game, mapEntry.canonical);
|
|
|
|
// Step 5: execute/Deploy — actually start the container.
|
|
db().transaction(() => {
|
|
deploymentStage(operation, 'Starting container', { phase: 'container-action',
|
|
checkpoint: { key: 'create-deployment', label: 'Deployment created and server registered' },
|
|
command: { state: 'attempted', message: 'Container startup submission attempted.' } });
|
|
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operation);
|
|
})();
|
|
const deployRes = await observeCommand(operation, () => komodoDeploy(name));
|
|
if (!deployRes.ok) {
|
|
if (deployRes.kind === 'unreachable' || (deployRes.httpStatus !== undefined && deployRes.httpStatus >= 500)) {
|
|
// A timeout or server error may arrive after the remote boot began.
|
|
// Preserve files, identity, and capacity until readiness is known.
|
|
recordDeploymentEvent(operation, 'Startup response uncertain; verifying the existing server: ' + deployRes.message);
|
|
recordProgress(operation, { command: { state: 'uncertain', message: 'Startup may have begun: ' + deployRes.message } });
|
|
return { ok: true, name, port, deploymentId: operation };
|
|
}
|
|
// Container never started — undo everything so retrying is clean.
|
|
recordProgress(operation, { command: { state: 'rejected', message: 'Startup request rejected: ' + deployRes.message } });
|
|
await komodoDeleteDeployment(name).catch(() => {});
|
|
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
|
|
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
|
|
return { ok: false, kind: 'komodo', message: `Deploy: ${deployRes.message}` };
|
|
}
|
|
|
|
recordProgress(operation, { command: { state: 'accepted', message: 'Komodo accepted startup; readiness still requires verification.' },
|
|
checkpoint: { key: 'container-action', label: 'Container startup accepted' } });
|
|
return { ok: true, name, port, deploymentId: operation };
|
|
}
|
|
|
|
/**
|
|
* Ensure `game` and `map` tags exist in Komodo, then attach both to the given
|
|
* deployment. Purely cosmetic — any failure here logs and returns; the caller
|
|
* treats it as fire-and-forget.
|
|
*/
|
|
async function applyDeploymentTags(deployment: string, gameTagName: string, mapTagName: string): Promise<void> {
|
|
try {
|
|
const [gameTag, mapTag] = await Promise.all([
|
|
komodoEnsureTag(gameTagName),
|
|
komodoEnsureTag(mapTagName)
|
|
]);
|
|
const tagIds: string[] = [];
|
|
if (gameTag.ok) tagIds.push(gameTag.id);
|
|
else console.warn(`[createServer] tag "${gameTagName}": ${gameTag.message}`);
|
|
if (mapTag.ok) tagIds.push(mapTag.id);
|
|
else console.warn(`[createServer] tag "${mapTagName}": ${mapTag.message}`);
|
|
if (tagIds.length === 0) return;
|
|
|
|
const res = await komodoUpdateTagsOnResource('Deployment', deployment, tagIds);
|
|
if (!res.ok) {
|
|
console.warn(`[createServer] UpdateResourceMeta (tags) for ${deployment}: ${res.message}`);
|
|
}
|
|
} catch (err) {
|
|
console.warn(`[createServer] tag application for ${deployment} threw:`, err);
|
|
}
|
|
}
|
|
|
|
const destroying = new Set<string>();
|
|
|
|
// Persist the request before returning so navigation and client disconnects cannot lose it.
|
|
function hasActiveCreation(name: string): boolean {
|
|
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name);
|
|
}
|
|
|
|
export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
|
|
const cancelled = cancelQueuedNativeCreation(name, triggeredBy);
|
|
if (cancelled) return { ok: true, warnings: [], deploymentId: cancelled };
|
|
const blocker = activeBetaRemovalBlocker(name);
|
|
if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] };
|
|
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild')
|
|
AND state IN ('waiting','queued','running')`).get(name)) {
|
|
return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] };
|
|
}
|
|
const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
|
|
if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
|
|
const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
|
|
if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id };
|
|
if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] };
|
|
const operation = db().transaction(() => {
|
|
const id = beginDeployment(name, 'destroy', triggeredBy);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
|
deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
|
|
blocker: { code: 'eligibility', message: 'Waiting for safe removal eligibility checks.', automatic: true } });
|
|
return id;
|
|
})();
|
|
return { ok: true, warnings: [], deploymentId: operation };
|
|
}
|
|
|
|
let destructionTick: Promise<void> | null = null;
|
|
export function processDestructions(): Promise<void> {
|
|
if (destructionTick) return destructionTick;
|
|
destructionTick = runDestructions().finally(() => { destructionTick = null; });
|
|
return destructionTick;
|
|
}
|
|
|
|
function betaRemovalEvidence(server: string, message: string): ProvisioningBlocker {
|
|
const beta = db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
|
|
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') ORDER BY b.id LIMIT 1`)
|
|
.get(server) as { id: number } | undefined;
|
|
return { code: 'active-beta', message, automatic: true, href: beta ? `/betas/${beta.id}` : null, label: 'Blocking beta' };
|
|
}
|
|
|
|
interface DestructionJob { id: string; server: string; stage: string; server_deleted_at: string | null }
|
|
|
|
async function runDestructions(): Promise<void> {
|
|
const queued = db().prepare(`SELECT id, server, stage, server_deleted_at FROM server_deployments WHERE action = 'destroy' AND state = 'waiting'
|
|
AND (server_deleted_at IS NULL OR stage = ?) ORDER BY started_at, rowid`).all(REMOVING_SERVER) as DestructionJob[];
|
|
const active = new Set<Promise<void>>();
|
|
let next = 0;
|
|
try {
|
|
while (next < queued.length || active.size) {
|
|
// Recheck the saved limit before each admission. Lowering it never
|
|
// interrupts an active removal; its slot lasts through local cleanup.
|
|
while (next < queued.length && !creating && destroying.size < fleetSettings().destroyConcurrency
|
|
&& active.size < fleetSettings().destroyConcurrency) {
|
|
const task = processDestruction(queued[next++]).finally(() => { active.delete(task); });
|
|
active.add(task);
|
|
}
|
|
if (!active.size) break; // Creation or rebuilding holds the remaining capacity.
|
|
await Promise.race(active);
|
|
}
|
|
} finally {
|
|
// Keep the shared pass locked until every admitted removal has settled,
|
|
// including when an unexpected persistence error escapes a task.
|
|
await Promise.allSettled(active);
|
|
}
|
|
}
|
|
|
|
async function processDestruction(operation: DestructionJob): Promise<void> {
|
|
if (nativeDeployment(operation.id)) return;
|
|
if (creating || destroying.has(operation.server)) return;
|
|
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) return;
|
|
destroying.add(operation.server);
|
|
try {
|
|
if (operation.stage === REMOVING_SERVER) {
|
|
if (operation.server_deleted_at && getServer(operation.server)) {
|
|
finishDeployment(operation.id, { ok: false, message: 'A replacement server is registered; local cleanup cannot safely resume.' });
|
|
return;
|
|
}
|
|
const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create'
|
|
AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id);
|
|
if (newer) {
|
|
finishDeployment(operation.id, { ok: false, message: 'A newer creation exists; local cleanup cannot safely resume.' });
|
|
return;
|
|
}
|
|
if (!claimRemoval(operation.id)) return;
|
|
finishDeployment(operation.id, await cleanupDestroyedServer(operation.server, operation.id));
|
|
return;
|
|
}
|
|
if (!getServer(operation.server)) {
|
|
if (hasActiveCreation(operation.server)) {
|
|
const creatingOperation = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL
|
|
AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health') LIMIT 1`).get(operation.server) as { id: string } | undefined;
|
|
deploymentStage(operation.id, 'Waiting for server creation to finish', { phase: 'eligibility', checks: [],
|
|
blocker: { code: 'creation-pending', message: 'Waiting for server creation to finish', automatic: true,
|
|
href: creatingOperation ? `/provisioning/${encodeURIComponent(creatingOperation.id)}` : null, label: 'Creation operation' } });
|
|
return;
|
|
}
|
|
finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' });
|
|
return;
|
|
}
|
|
const blocker = activeBetaRemovalBlocker(operation.server);
|
|
if (blocker) { deploymentStage(operation.id, blocker, { phase: 'eligibility', checks: [], blocker: betaRemovalEvidence(operation.server, blocker) }); return; }
|
|
// A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
|
|
const evidence = await probePlayerEvidence(operation.server);
|
|
let players = evidence.players;
|
|
let occupancy: NonNullable<ProgressUpdate['checks']>[number] = { key: 'occupancy', label: 'Player occupancy',
|
|
result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting', source: evidence.source,
|
|
checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players };
|
|
if (players === null) {
|
|
const health = await komodoInspectHealthy(operation.server);
|
|
if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) {
|
|
players = 0;
|
|
occupancy = { key: 'stopped', label: 'Stopped container permits removal', result: 'pass', source: 'komodo', actual: health.state ?? null };
|
|
}
|
|
}
|
|
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) return;
|
|
// Beta ownership may have changed while the network probes were in flight.
|
|
const latestBlocker = activeBetaRemovalBlocker(operation.server);
|
|
if (latestBlocker) { deploymentStage(operation.id, latestBlocker, { phase: 'eligibility', checks: [occupancy], blocker: betaRemovalEvidence(operation.server, latestBlocker) }); return; }
|
|
if (players !== 0) {
|
|
const message = players === null
|
|
? 'Waiting for player count: server could not be reached'
|
|
: 'Waiting for server to empty: ' + players + ' player(s) online';
|
|
deploymentStage(operation.id, message, { phase: 'eligibility', checks: [occupancy],
|
|
blocker: { code: players === null ? 'players-unknown' : 'players-present', message, automatic: true } });
|
|
return;
|
|
}
|
|
if (!claimRemoval(operation.id)) return;
|
|
deploymentStage(operation.id, 'Server is empty (0 players verified)', { phase: 'eligibility', checks: [occupancy], blocker: null,
|
|
checkpoint: { key: 'eligibility', label: 'Removal eligibility verified' } });
|
|
const result = await destroyServerUnlocked(operation.server, operation.id);
|
|
finishDeployment(operation.id, result);
|
|
} catch (err) {
|
|
const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string };
|
|
if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message, {
|
|
phase: 'eligibility', checks: [], blocker: { code: 'players-unknown', message: (err as Error).message, automatic: true }
|
|
});
|
|
else if (row.state !== 'cancelled') finishDeployment(operation.id, { ok: false, message: (err as Error).message });
|
|
} finally { destroying.delete(operation.server); }
|
|
}
|
|
|
|
async function destroyServerUnlocked(name: string, operation: string): Promise<DestroyServerResult> {
|
|
const blocker = activeBetaRemovalBlocker(name);
|
|
if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] };
|
|
const row = getServer(name);
|
|
if (!row) {
|
|
return { ok: false, kind: 'not-found', message: `no server named '${name}'`, warnings: [] };
|
|
}
|
|
|
|
const warnings: string[] = [];
|
|
|
|
// 1: stop + remove the container. execute/DestroyDeployment queues the
|
|
// teardown; the response comes back before the container is actually gone.
|
|
deploymentStage(operation, 'Stopping and removing container', { phase: 'remove-container', blocker: null, checks: [],
|
|
checkpoint: { key: 'eligibility', label: 'Removal eligibility verified' },
|
|
command: { state: 'attempted', message: 'Container removal submission attempted.' } });
|
|
const destroyRes = await observeCommand(operation, () => komodoDestroyDeployment(name));
|
|
if (!destroyRes.ok) {
|
|
recordProgress(operation, { command: { state: destroyRes.kind === 'unreachable' || (destroyRes.httpStatus !== undefined && destroyRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: destroyRes.message } });
|
|
return { ok: false, kind: 'komodo', message: 'DestroyDeployment failed: ' + destroyRes.message, warnings };
|
|
}
|
|
recordProgress(operation, { command: { state: 'accepted', message: 'Komodo accepted teardown; removal is not yet confirmed.' } });
|
|
|
|
// 2: wait for the container to actually disappear. Komodo refuses to
|
|
// DeleteDeployment while a container still exists for it, so poll the
|
|
// health probe until it stops reporting a running state. `ok:false` here
|
|
// is the *success* signal — it means the container is no longer inspectable.
|
|
const pollDeadline = Date.now() + 30_000;
|
|
while (Date.now() < pollDeadline) {
|
|
const health = await komodoInspectHealthy(name);
|
|
if (!health.ok) break;
|
|
await new Promise((resolve) => setTimeout(resolve, 1500));
|
|
}
|
|
|
|
// 3: remove the deployment entity itself.
|
|
deploymentStage(operation, 'Deleting Komodo deployment', { phase: 'delete-deployment',
|
|
command: { state: 'attempted', message: 'Komodo deployment deletion attempted.' } });
|
|
const deleteRes = await observeCommand(operation, () => komodoDeleteDeployment(name));
|
|
if (!deleteRes.ok) {
|
|
recordProgress(operation, { command: { state: deleteRes.kind === 'unreachable' || (deleteRes.httpStatus !== undefined && deleteRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: deleteRes.message } });
|
|
return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings };
|
|
}
|
|
|
|
deploymentStage(operation, REMOVING_SERVER, { phase: 'cleanup', blocker: null, checks: [],
|
|
command: { state: 'accepted', message: 'Komodo confirmed deployment deletion.' },
|
|
checkpoints: [{ key: 'remove-container', label: 'Container removal confirmed by deployment deletion' },
|
|
{ key: 'delete-deployment', label: 'Komodo deployment deletion confirmed' }] });
|
|
|
|
return cleanupDestroyedServer(name, operation);
|
|
}
|
|
|
|
async function cleanupDestroyedServer(name: string, operation: string): Promise<DestroyServerResult> {
|
|
const blocker = activeBetaRemovalBlocker(name);
|
|
if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] };
|
|
const row = getServer(name);
|
|
const warnings: string[] = [];
|
|
// Persist the checkpoint before local work so failed cleanup can resume without RCON.
|
|
// Legacy operations already recorded this same stage after Komodo deletion.
|
|
deploymentStage(operation, REMOVING_SERVER, { phase: 'cleanup', blocker: null, checks: [],
|
|
checkpoint: { key: 'delete-deployment', label: 'Komodo deployment deletion confirmed' } });
|
|
// Host directory. Fall back to derived path if host_dir wasn't set on
|
|
// legacy rows.
|
|
const hostDir = row?.host_dir ?? serverDir(name);
|
|
try {
|
|
await rm(hostDir, { recursive: true, force: true });
|
|
recordProgress(operation, { checkpoint: { key: 'files-removed', label: 'Instance files removed', phase: 'cleanup' } });
|
|
} catch (err) {
|
|
warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
|
|
}
|
|
|
|
// The registry trigger preserves history and marks its old server identity.
|
|
// Live inventory, rollback snapshots, and restart slots still cascade away.
|
|
try {
|
|
db().transaction(() => {
|
|
const recovery = db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ? AND healthy_at IS NULL').get(name) as { operation_id: string | null } | undefined;
|
|
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
|
|
if (recovery?.operation_id) recordProgress(recovery.operation_id, { recovery: { state: 'released', operationId: recovery.operation_id,
|
|
message: 'Recovery capacity released when the old instance registry was removed.' } });
|
|
recordProgress(operation, { checkpoint: { key: 'registry-removed', label: 'Server registration removed', phase: 'cleanup' } });
|
|
if (!warnings.length) recordProgress(operation, { checkpoint: { key: 'cleanup', label: 'Local files and registry cleanup completed' } });
|
|
})();
|
|
} catch (err) {
|
|
return {
|
|
ok: false,
|
|
kind: 'db',
|
|
message: `failed to remove DB rows for ${name}: ${(err as Error).message}`,
|
|
warnings
|
|
};
|
|
}
|
|
|
|
const unfinished = db().prepare("SELECT id, action FROM server_deployments WHERE server = ? AND state = 'awaiting-health'").all(name) as { id: string; action: string }[];
|
|
for (const { id, action } of unfinished) finishDeployment(id, { ok: false, message: action === 'create'
|
|
? 'Server destroyed before creation health checks completed' : 'Server destroyed before operation verification completed' });
|
|
return { ok: true, warnings };
|
|
}
|
|
interface CreationJob {
|
|
deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
|
|
submitted_at?: string | null;
|
|
}
|
|
|
|
async function provisionCreation(id: string): Promise<CreateServerResult> {
|
|
const job = db().prepare('SELECT j.*, d.triggered_by FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE j.deployment_id = ?').get(id) as CreationJob;
|
|
creating = true;
|
|
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(id);
|
|
try {
|
|
const mapEntry = zombieMapBySlug(gameOfServer(job.server), job.map_slug);
|
|
const plutoKey = getServerKey(job.server);
|
|
if (!mapEntry || !plutoKey) {
|
|
const result = { ok: false as const, kind: 'bad-request' as const, message: 'Saved server key or selected map is no longer available' };
|
|
finishDeployment(id, result);
|
|
return result;
|
|
}
|
|
const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by },
|
|
{ ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
|
|
if (!result.ok) finishDeployment(id, result);
|
|
else {
|
|
db().transaction(() => {
|
|
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
|
deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', checks: [],
|
|
blocker: { code: 'container-readiness', message: 'Waiting for container readiness.', automatic: true } });
|
|
})();
|
|
}
|
|
return result;
|
|
} catch (err) {
|
|
const server = getServer(job.server);
|
|
if (server?.game_port === job.port) {
|
|
// A lost response may have happened after Komodo started the server.
|
|
// Keep its slot and observe readiness instead of starting another boot.
|
|
db().transaction(() => {
|
|
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
|
recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message);
|
|
deploymentStage(id, 'Verifying existing server after interrupted startup', { phase: 'verify-container', checks: [],
|
|
command: { state: 'uncertain', message: 'Startup response interrupted: ' + (err as Error).message },
|
|
blocker: { code: 'container-readiness', message: 'Observing the existing instance after an uncertain startup.', automatic: true } });
|
|
})();
|
|
} else finishDeployment(id, { ok: false, message: (err as Error).message });
|
|
throw err;
|
|
} finally { creating = false; }
|
|
}
|
|
|
|
type CreationOutcomes = Map<string, CreateServerResult | Error>;
|
|
let creationTick: Promise<CreationOutcomes> | null = null;
|
|
function creationPass(): Promise<CreationOutcomes> {
|
|
if (creationTick) return creationTick;
|
|
creationTick = runCreations().finally(() => { creationTick = null; });
|
|
return creationTick;
|
|
}
|
|
|
|
export async function processCreations(): Promise<void> {
|
|
await creationPass();
|
|
}
|
|
|
|
/** One host-wide FIFO, shared by collections and individual launches. */
|
|
export function creationQueue() {
|
|
type Entry = { id: string; server: string; state: string; stage: string };
|
|
const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments
|
|
WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
|
|
ORDER BY started_at, rowid`).all() as Entry[];
|
|
return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
|
|
}
|
|
|
|
function creationBlocker(id: string): ProvisioningBlocker | null {
|
|
const dependency = db().prepare(`SELECT d.id, d.server, d.state FROM server_creation_dependencies dep
|
|
JOIN server_deployments d ON d.id = dep.removal_id
|
|
WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { id: string; server: string; state: string } | undefined;
|
|
if (dependency) return { code: 'removal-dependency', message: `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`,
|
|
automatic: true, href: `/provisioning/${encodeURIComponent(dependency.id)}`, label: 'Required removal' };
|
|
const occupied = db().prepare(`SELECT s.name FROM servers s JOIN server_creation_jobs j ON s.game_port = j.port
|
|
WHERE j.deployment_id = ? LIMIT 1`).get(id) as { name: string } | undefined;
|
|
return occupied ? { code: 'port-occupied', message: `Waiting for port to be released by ${occupied.name}`, automatic: true,
|
|
href: `/servers/${encodeURIComponent(occupied.name)}`, label: 'Server using this port' } : null;
|
|
}
|
|
|
|
function stillCheckingCreation(id: string): boolean {
|
|
return !!db().prepare(`SELECT 1 FROM server_deployments d JOIN server_creation_jobs j ON j.deployment_id = d.id
|
|
JOIN servers s ON s.name = j.server AND s.game_port = j.port
|
|
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state = 'awaiting-health'`).get(id);
|
|
}
|
|
|
|
async function runCreations(): Promise<CreationOutcomes> {
|
|
const outcomes: CreationOutcomes = new Map();
|
|
if (creating || destroying.size > 0) {
|
|
for (const job of creationQueue().queued) recordProgress(job.id, { phase: 'startup-queue', blocker: {
|
|
code: 'startup-worker-busy', message: 'Waiting for in-flight instance creation or removal to finish.', automatic: true
|
|
} });
|
|
return outcomes;
|
|
}
|
|
// Restart recovery is observation-only: never repeat external creation or
|
|
// deployment. A known interrupted startup must retain its readiness gate.
|
|
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
|
|
for (const { id } of interrupted) {
|
|
if (nativeDeployment(id)) continue;
|
|
if (resumeVerification(id)) continue;
|
|
// A queued removal does not make the interrupted container disappear.
|
|
// Keep its slot until removal finishes, without resuming normal work or
|
|
// confusing it with a newer creation of the same server identity.
|
|
const removing = db().prepare(`SELECT 1 FROM server_deployments d
|
|
JOIN server_creation_jobs j ON j.deployment_id = d.id
|
|
JOIN servers s ON s.name = j.server AND s.game_port = j.port
|
|
WHERE d.id = ? AND EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server
|
|
AND newer.rowid > d.rowid AND newer.action = 'destroy' AND newer.state IN ('waiting', 'running'))
|
|
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server
|
|
AND newer.rowid > d.rowid AND newer.action = 'create')`).get(id);
|
|
if (removing) {
|
|
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
|
deploymentStage(id, 'Waiting for requested server removal after interrupted startup', { checks: [],
|
|
blocker: { code: 'removal-pending', message: 'Waiting for the requested removal; the interrupted startup still occupies capacity.', automatic: true } });
|
|
}
|
|
}
|
|
|
|
// Check only previously started servers. Newly launched servers keep their
|
|
// slot until a later pass, even if a health probe would answer immediately.
|
|
const checking = db().prepare(`SELECT j.*, COALESCE(d.submitted_at, d.started_at) AS submitted_at FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
|
WHERE d.server_deleted_at IS NULL AND d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[];
|
|
for (const job of checking) {
|
|
const id = job.deployment_id;
|
|
if (nativeDeployment(id)) continue;
|
|
if (getServer(job.server)?.game_port !== job.port) {
|
|
finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' });
|
|
continue;
|
|
}
|
|
const removal = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server) as { id: string } | undefined;
|
|
if (removal) {
|
|
recordProgress(id, { checks: [], blocker: { code: 'removal-pending', message: 'Creation verification is held while server removal is pending.', automatic: true,
|
|
href: `/provisioning/${encodeURIComponent(removal.id)}`, label: 'Removal operation' } });
|
|
continue;
|
|
}
|
|
const checks: NonNullable<ProgressUpdate['checks']> = [];
|
|
let verifyingPlutonium = false;
|
|
try {
|
|
const komodo = await komodoInspectHealthy(job.server);
|
|
if (!stillCheckingCreation(id)) continue;
|
|
checks.push({ key: 'container', label: 'Container readiness', source: 'komodo', result: komodo.ok ? 'pass' : 'waiting',
|
|
checkedAt: new Date().toISOString(), actual: komodo.state ?? null, message: komodo.ok ? null : komodo.reason });
|
|
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason, { phase: 'verify-container', checks,
|
|
blocker: { code: 'container-readiness', message: komodo.reason, automatic: true } }); continue; }
|
|
const status = await probeServerStatus(job.server, job.submitted_at);
|
|
if (!stillCheckingCreation(id)) continue;
|
|
checks.push({ key: 'game', label: 'Game status', source: status.ok ? status.source : null, result: status.ok ? 'pass' : 'waiting',
|
|
checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null, message: status.ok ? null : status.reason });
|
|
if (!status.ok) { deploymentStage(id, 'Waiting for RCON or fresh telemetry health: ' + status.reason, { phase: 'verify-game', checks,
|
|
blocker: { code: 'game-readiness', message: status.reason, automatic: true } }); continue; }
|
|
const actualMap = status.map;
|
|
checks.push({ key: 'map', label: 'Selected engine map', source: status.source, result: actualMap === job.expected_map ? 'pass' : 'waiting',
|
|
checkedAt: status.observedAt, receivedAt: status.receivedAt, expected: job.expected_map, actual: actualMap });
|
|
if (actualMap !== job.expected_map) {
|
|
const message = 'Waiting for selected map: expected ' + job.expected_map + ', ' + (status.source === 'rcon' ? 'RCON' : 'telemetry') + ' reports ' + (actualMap ?? 'no map');
|
|
deploymentStage(id, message, { phase: 'verify-map', checks, blocker: { code: 'map-mismatch', message, automatic: true,
|
|
href: `/servers/${encodeURIComponent(job.server)}?tab=config`, label: 'Map configuration' } });
|
|
continue;
|
|
}
|
|
verifyingPlutonium = true;
|
|
recordProgress(id, { phase: 'verify-plutonium', checks });
|
|
// Refresh only the release, then probe this instance. Fleet-wide checks
|
|
// can run independently and must not be required to complete a launch.
|
|
await checkRelease();
|
|
if (!stillCheckingCreation(id)) continue;
|
|
const revision = await probeVersion(job.server);
|
|
if (!stillCheckingCreation(id)) continue;
|
|
const target = freshRelease();
|
|
const published = release();
|
|
const version = db().prepare('SELECT checked_at, error FROM plutonium_versions WHERE server = ?').get(job.server) as
|
|
{ checked_at: string; error: string | null } | undefined;
|
|
checks.push({ key: 'plutonium-release', label: 'Published Plutonium revision', source: 'manifest',
|
|
result: target === null ? 'unknown' : 'pass', checkedAt: published.checkedAt ?? undefined, actual: target,
|
|
message: target === null ? published.error ?? 'The production revision could not be verified.' : null });
|
|
checks.push({ key: 'plutonium-version', label: 'Running Plutonium revision', source: 'rcon',
|
|
result: revision === null || target === null ? 'unknown' : revision < target ? 'fail' : 'pass',
|
|
checkedAt: version?.checked_at, expected: target, actual: revision, message: version?.error });
|
|
if (target === null || revision === null) {
|
|
const message = target === null ? 'Waiting for the Plutonium production manifest: ' + (published.error ?? 'revision unavailable')
|
|
: 'Waiting for the running Plutonium revision: ' + (version?.error ?? 'version unavailable');
|
|
deploymentStage(id, message, { phase: 'verify-plutonium', checks,
|
|
blocker: { code: target === null ? 'plutonium-release' : 'plutonium-version', message, automatic: true } });
|
|
continue;
|
|
}
|
|
if (revision < target) {
|
|
const message = `Plutonium verification failed: running r${revision}, production is r${target}. Inspect the startup updater before retrying.`;
|
|
db().transaction(() => {
|
|
deploymentStage(id, message, { phase: 'verify-plutonium', checks, blocker: null });
|
|
finishDeployment(id, { ok: false, message });
|
|
})();
|
|
continue;
|
|
}
|
|
db().transaction(() => {
|
|
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry')
|
|
+ `, map ${actualMap}, and Plutonium r${revision} (production r${target})`, {
|
|
phase: 'verify-plutonium', checks, blocker: null, checkpoints: [
|
|
{ key: 'container-action', label: 'Container startup verified' },
|
|
{ key: 'verify-container', label: 'Container readiness verified' },
|
|
{ key: 'verify-game', label: 'Game status verified' },
|
|
{ key: 'verify-map', label: 'Selected engine map verified' },
|
|
{ key: 'verify-plutonium', label: `Plutonium r${revision} verified against production r${target}` }
|
|
]
|
|
});
|
|
finishDeployment(id, { ok: true });
|
|
})();
|
|
} catch (err) {
|
|
if (!stillCheckingCreation(id)) continue;
|
|
const key = verifyingPlutonium ? 'plutonium-version' : checks.length ? 'game' : 'container';
|
|
const remaining = checks.filter(check => check.key !== key);
|
|
remaining.push({ key, label: verifyingPlutonium ? 'Running Plutonium revision' : checks.length ? 'Game status' : 'Container readiness',
|
|
source: null, result: 'unknown', message: (err as Error).message });
|
|
deploymentStage(id, 'Waiting for verification: ' + (err as Error).message, {
|
|
phase: verifyingPlutonium ? 'verify-plutonium' : checks.length ? 'verify-game' : 'verify-container', checks: remaining,
|
|
blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true } });
|
|
}
|
|
}
|
|
|
|
// Snapshot the budget once. Fast success/failure must not turn a single
|
|
// worker pass into an unbounded stream of expensive container startups.
|
|
const queue = creationQueue();
|
|
const available = Math.max(0, fleetSettings().createConcurrency - queue.active.length);
|
|
const eligible = queue.queued.filter(({ id }) => {
|
|
if (nativeDeployment(id)) return false;
|
|
const blocker = creationBlocker(id);
|
|
if (blocker) deploymentStage(id, blocker.message, { phase: 'startup-queue', blocker });
|
|
else recordProgress(id, { phase: 'startup-queue', blocker: { code: available ? 'startup-queue' : 'startup-capacity',
|
|
message: available ? 'Eligible for startup admission in request order.' : 'Waiting for startup capacity to be released.', automatic: true } });
|
|
return !blocker;
|
|
});
|
|
for (const { id } of eligible.slice(0, available)) {
|
|
if (creating || destroying.size > 0) break;
|
|
// API calls can cancel a selected job or resume an older startup while
|
|
// a previous provision awaits Komodo. Recheck without expanding this pass.
|
|
if (creationQueue().active.length >= fleetSettings().createConcurrency) break;
|
|
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
|
|
const blocker = creationBlocker(id);
|
|
if (blocker) {
|
|
deploymentStage(id, blocker.message, { phase: 'startup-queue', blocker });
|
|
continue;
|
|
}
|
|
try { outcomes.set(id, await provisionCreation(id)); }
|
|
catch (err) {
|
|
const failure = err instanceof Error ? err : new Error(String(err));
|
|
outcomes.set(id, failure);
|
|
console.error('[creation] provisioning failed:', failure);
|
|
}
|
|
}
|
|
return outcomes;
|
|
}
|