refactor: make provisioning own instance lifecycle operations
This commit is contained in:
@@ -0,0 +1,796 @@
|
||||
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
|
||||
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { LaunchPreview } from '$lib/deployment-plan.js';
|
||||
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
|
||||
import { mkdir, chmod, rm, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
|
||||
import { getServer } from '../fleet/servers.js';
|
||||
import { zombieMapBySlug, type GameMode, type ZombieMapEntry, type ZombieGame } from '../fleet/map-aliases.js';
|
||||
import {
|
||||
komodoCreateDeployment,
|
||||
komodoDeploy,
|
||||
komodoDestroyDeployment,
|
||||
komodoDeleteDeployment,
|
||||
komodoInspectHealthy,
|
||||
komodoEnsureTag,
|
||||
komodoUpdateTagsOnResource,
|
||||
type CreateDeploymentSpec,
|
||||
type DeploymentEnvVar
|
||||
} from '../integrations/komodo.js';
|
||||
import { MAP_CONFIG_FILE } from '../fleet/server-config.js';
|
||||
import { recordScript } from '../fleet/inventory.js';
|
||||
import { getVersion } from '../fleet/scripts.js';
|
||||
|
||||
export type CreateServerFailure =
|
||||
| 'bad-request'
|
||||
| 'unconfigured' // Komodo creds / target server missing
|
||||
| 'unknown-map' // slug isn't in the catalog
|
||||
| 'no-slot' // all lobby slots for this map are taken
|
||||
| 'no-port' // port range exhausted
|
||||
| 'fs' // couldn't create host dir
|
||||
| 'komodo' // Komodo CreateDeployment / Deploy rejected
|
||||
| 'db'; // shouldn't happen, but wraps INSERT failures
|
||||
|
||||
export type CreateServerResult =
|
||||
| { ok: true; name: string; port: number; deploymentId: string }
|
||||
| { ok: false; kind: CreateServerFailure; message: string };
|
||||
|
||||
export type DestroyServerResult =
|
||||
| { ok: true; warnings: string[] }
|
||||
| { ok: false; kind: 'not-found' | 'conflict' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] };
|
||||
|
||||
export interface CreateServerInput {
|
||||
game?: ZombieGame; // omitted by older clients: T5
|
||||
gameMode: GameMode; // 'zombie' — only value supported today
|
||||
mapSlug: string; // 'cosmodrome'
|
||||
plutoKey?: string; // saved for this server ID when supplied
|
||||
expectedName?: string; // optional preview guard; never selects a lobby
|
||||
previewId?: string;
|
||||
createdBy?: string; // 'ui' by default
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the lowest lobby number `n` such that `t5_<mode>_<slug>_<n>` isn't in
|
||||
* the servers table. Caps at 20 to stay inside the port range — with 20 total
|
||||
* ports across the whole fleet, no one map can hold more than that.
|
||||
*/
|
||||
export function nextLobbyNumber(baseName: string, additionalNames: string[] = []): number | null {
|
||||
const rows = db()
|
||||
.prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
|
||||
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
|
||||
.all(`${baseName}_%`) as Array<{ name: string }>;
|
||||
const taken = new Set<number>();
|
||||
const prefix = `${baseName}_`;
|
||||
for (const r of [...rows, ...additionalNames.map(name => ({ name }))]) {
|
||||
if (!r.name.startsWith(prefix)) continue;
|
||||
const tail = r.name.slice(prefix.length);
|
||||
if (/^\d+$/.test(tail)) taken.add(Number.parseInt(tail, 10));
|
||||
}
|
||||
for (let n = 1; n <= 20; n++) {
|
||||
if (!taken.has(n)) return n;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lowest unused port in the configured range. Reads every server's game_port
|
||||
* from the DB (which EC controls end-to-end under the new naming scheme, so
|
||||
* the DB is authoritative — no need to ask Komodo).
|
||||
*/
|
||||
export function nextAvailablePort(additionalPorts: number[] = [], releasingServers: string[] = []): number | null {
|
||||
const rows = db()
|
||||
.prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
|
||||
SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
|
||||
.all() as Array<{ name: string; game_port: number }>;
|
||||
const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
|
||||
for (let p = config.portRange.min; p <= config.portRange.max; p++) {
|
||||
if (!taken.has(p)) return p;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function buildEnv(args: {
|
||||
game: ZombieGame;
|
||||
name: string;
|
||||
port: number;
|
||||
plutoKey: string;
|
||||
}): DeploymentEnvVar[] {
|
||||
return [
|
||||
{ variable: 'PLUTAINER_SERVER_NAME', value: args.name },
|
||||
{ variable: 'PLUTAINER_PORT', value: String(args.port) },
|
||||
{ variable: 'PLUTAINER_GAME', value: args.game === 't6' ? 't6zm' : 't5sp' },
|
||||
{ variable: 'PLUTAINER_CONFIG_FILE', value: configFileForGame(args.game) },
|
||||
// Plutainer seeds/loads the main config first. Load our generated map
|
||||
// config afterward, then rotate. A separate exec avoids quoted values
|
||||
// in EXTRA_ARGS, which Plutainer splits on whitespace.
|
||||
{ variable: 'PLUTAINER_MAP_ROTATE', value: 'false' },
|
||||
// Give telemetry scripts the registry name before the first map starts.
|
||||
{ variable: 'PLUTAINER_EXTRA_ARGS', value: `+exec ${MAP_CONFIG_FILE} +set tws_server_id ${args.name} +map_rotate` },
|
||||
{ variable: 'PLUTO_SERVER_KEY', value: args.plutoKey },
|
||||
{ variable: 'PLUTAINER_RCON_PASSWORD', value: config.rcon.password }
|
||||
];
|
||||
}
|
||||
|
||||
function buildSpec(args: {
|
||||
game: ZombieGame;
|
||||
name: string;
|
||||
port: number;
|
||||
hostDir: string;
|
||||
plutoKey: string;
|
||||
recipe: LaunchRecipe;
|
||||
}): CreateDeploymentSpec {
|
||||
return {
|
||||
name: args.name,
|
||||
server: config.komodo.targetServer,
|
||||
image: args.recipe.image,
|
||||
network: config.komodo.defaultNetwork,
|
||||
// Container listens on PLUTAINER_PORT (== args.port), so host and
|
||||
// container ports match — no NAT inside the bridge network.
|
||||
ports: [
|
||||
`${args.port}:${args.port}/udp`,
|
||||
`${args.port}:${args.port}/tcp`
|
||||
],
|
||||
volumes: [
|
||||
`${args.hostDir}:/home/plutainer/app`,
|
||||
`${args.game === 't6' ? config.komodo.t6GameFilesVolume : config.komodo.gameFilesVolume}:/home/plutainer/gamefiles:ro`
|
||||
],
|
||||
environment: buildEnv(args)
|
||||
};
|
||||
}
|
||||
|
||||
let creating = false;
|
||||
interface PreparedCreation {
|
||||
ok: true; name: string; port: number; plutoKey: string; mapEntry: ZombieMapEntry;
|
||||
}
|
||||
|
||||
interface PreviewRow {
|
||||
id: string; expires_at: string; server: string | null; port: number | null;
|
||||
recipe: string; blueprint_name: string | null; deployment_id: string | null;
|
||||
}
|
||||
|
||||
/** Read-only checks; a preview never reserves a slot or writes credentials. */
|
||||
export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown }): LaunchPreview {
|
||||
if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.');
|
||||
let recipe: LaunchRecipe;
|
||||
let blueprintName: string | null = null;
|
||||
if (input.blueprintId !== undefined) {
|
||||
if (typeof input.blueprintId !== 'string') throw new Error('Invalid blueprint.');
|
||||
({ recipe, blueprintName } = blueprintRecipe(input.blueprintId));
|
||||
} else if (input.deploymentId !== undefined) {
|
||||
if (typeof input.deploymentId !== 'string') throw new Error('Invalid deployment.');
|
||||
const source = recipeForDeployment(input.deploymentId);
|
||||
if (!source) throw new Error('This deployment has no recorded launch recipe.');
|
||||
blueprintName = source.blueprintName;
|
||||
// History describes the previous launch; only its map selection is reusable.
|
||||
recipe = captureRecipe(source.recipe.game, source.recipe.mapSlug);
|
||||
} else {
|
||||
if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.');
|
||||
recipe = captureRecipe(input.game, input.mapSlug);
|
||||
}
|
||||
const lobby = nextLobbyNumber(`${recipe.game}_zombie_${recipe.mapSlug}`);
|
||||
const name = lobby === null ? null : `${recipe.game}_zombie_${recipe.mapSlug}_${lobby}`;
|
||||
const port = nextAvailablePort();
|
||||
const configured = !!(config.komodo.address && config.komodo.apiKey && config.komodo.apiSecret && config.komodo.targetServer);
|
||||
const checks: LaunchPreview['checks'] = [
|
||||
{ label: 'Komodo settings', state: configured ? 'pass' : 'blocked', detail: configured ? 'Connection settings and target host are configured.' : 'Configure Komodo credentials and a target host before launching.' },
|
||||
{ label: 'Health verification', state: config.rcon.password ? 'pass' : 'blocked', detail: config.rcon.password ? 'RCON is configured. Container, RCON, and map checks run after launch.' : 'Configure RCON_PASSWORD so the new server can be verified.' },
|
||||
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All 20 lobbies for this map are occupied or reserved.' },
|
||||
{ label: 'Game port', state: port === null ? 'blocked' : 'pass', detail: port === null ? 'No free game port. Free capacity before launching.' : `${port} / UDP and TCP. Availability is checked again at launch.` },
|
||||
{ label: 'Promoted configuration', state: 'pass', detail: recipe.configSource },
|
||||
{ label: 'Game files and host readiness', state: 'info', detail: 'The host must have the selected game and DLC files. This preview does not probe volumes, images, credentials, or network access.' }
|
||||
];
|
||||
const id = randomUUID();
|
||||
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
||||
db().prepare('DELETE FROM server_launch_previews WHERE expires_at < ? AND deployment_id IS NULL').run(new Date().toISOString());
|
||||
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
|
||||
return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName,
|
||||
checks, ready: checks.every(check => check.state !== 'blocked') };
|
||||
}
|
||||
|
||||
// releasingServers is internal to atomic collection queueing; HTTP callers cannot set it.
|
||||
export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult {
|
||||
return db().transaction((): CreateServerResult => {
|
||||
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined;
|
||||
if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' };
|
||||
const recipe: LaunchRecipe | undefined = preview ? JSON.parse(preview.recipe) : undefined;
|
||||
if (recipe && (recipe.game !== (input.game ?? 't5') || recipe.mapSlug !== input.mapSlug || input.gameMode !== 'zombie')) {
|
||||
return { ok: false, kind: 'bad-request', message: 'The selection does not match the reviewed launch recipe.' };
|
||||
}
|
||||
// A repeated submit after a lost response returns the same operation.
|
||||
if (preview?.deployment_id) return { ok: true, name: preview.server!, port: preview.port!, deploymentId: preview.deployment_id };
|
||||
if (preview && preview.expires_at <= new Date().toISOString()) return { ok: false, kind: 'bad-request', message: 'Launch preview expired. Review a fresh preview.' };
|
||||
if (preview && (!preview.server || !preview.port || preview.server !== nextName(input) || preview.port !== nextAvailablePort([], releasingServers))) {
|
||||
return { ok: false, kind: 'no-slot', message: 'Available lobby or port changed. Refresh the preview and review the new server key.' };
|
||||
}
|
||||
if (recipe && !config.rcon.password) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'Health-check settings changed. Refresh the preview.' };
|
||||
}
|
||||
const prepared = prepareCreation(input, releasingServers);
|
||||
if (!prepared.ok) return prepared;
|
||||
// A preview records choices, not version pins. Promotions remain authoritative.
|
||||
const currentRecipe = captureRecipe(input.game ?? 't5', prepared.mapEntry.slug);
|
||||
const id = beginDeployment(prepared.name, 'create', input.createdBy);
|
||||
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, blueprint_name) VALUES (?, ?, ?)')
|
||||
.run(id, JSON.stringify(currentRecipe), preview?.blueprint_name ?? null);
|
||||
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id = ? WHERE id = ?').run(id, preview.id);
|
||||
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical);
|
||||
deploymentStage(id, 'Queued for server creation');
|
||||
return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id };
|
||||
})();
|
||||
}
|
||||
|
||||
/** Reserve the same identity and port before removing any part of the instance. */
|
||||
export function queueRebuildServer(name: string): CreateServerResult {
|
||||
const server = getServer(name);
|
||||
const game = gameOfServer(name);
|
||||
const map = server?.map_slug && zombieMapBySlug(game, server.map_slug);
|
||||
if (!server || !map || !server.game_port || server.komodo_type !== 'deployment' || server.komodo_name !== name
|
||||
|| (server.host_dir && server.host_dir !== serverDir(name))) {
|
||||
return { ok: false, kind: 'bad-request', message: 'Rebuild requires a managed server with its original map, directory, and port.' };
|
||||
}
|
||||
if (!config.komodo.targetServer || !config.rcon.password) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' };
|
||||
if (!getServerKey(name)) return { ok: false, kind: 'bad-request', message: `Save a server key for ${name} in Keys before rebuilding.` };
|
||||
// Validate that the replacement setup can be composed before destruction.
|
||||
const recipe = captureRecipe(game, map.slug);
|
||||
return db().transaction(() => {
|
||||
const id = beginDeployment(name, 'rebuild');
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
|
||||
db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
|
||||
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe));
|
||||
deploymentStage(id, 'Waiting to rebuild the empty server');
|
||||
return { ok: true as const, name, port: server.game_port!, deploymentId: id };
|
||||
})();
|
||||
}
|
||||
|
||||
/** Caller has just verified emptiness and excluded conflicting work. */
|
||||
export async function rebuildServer(id: string): Promise<void> {
|
||||
const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.id = ? AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
|
||||
if (!job) return;
|
||||
if (creating || destroying.size || creationQueue().active.length >= config.createConcurrency) {
|
||||
// The cleanup checkpoint must survive capacity waits and app restarts.
|
||||
if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding');
|
||||
return;
|
||||
}
|
||||
// Recheck prerequisites while the original instance still exists.
|
||||
if (!getServerKey(job.server)) { finishDeployment(id, { ok: false, message: 'Saved server key was removed; rebuild did not begin.' }); return; }
|
||||
try { captureRecipe(gameOfServer(job.server), job.map_slug); }
|
||||
catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); return; }
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
|
||||
destroying.add(job.server);
|
||||
try {
|
||||
const result = job.stage === REMOVING_SERVER
|
||||
? await cleanupDestroyedServer(job.server, id) : await destroyServerUnlocked(job.server, id);
|
||||
if (!result.ok || result.warnings.length) {
|
||||
finishDeployment(id, { ok: false, message: result.ok ? 'Old server files could not be completely removed.' : result.message, warnings: result.warnings });
|
||||
return;
|
||||
}
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE server_rebuild_jobs SET removed_at = ? WHERE deployment_id = ?').run(new Date().toISOString(), id);
|
||||
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Queued for server creation');
|
||||
})();
|
||||
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
|
||||
finally { destroying.delete(job.server); }
|
||||
}
|
||||
|
||||
function nextName(input: CreateServerInput): string | null {
|
||||
const base = `${input.game ?? 't5'}_${input.gameMode}_${input.mapSlug}`;
|
||||
const lobby = nextLobbyNumber(base);
|
||||
return lobby === null ? null : `${base}_${lobby}`;
|
||||
}
|
||||
|
||||
// Internal callers use the same durable queue and capacity limit as HTTP launches.
|
||||
// Await this pass's provisioning when possible; a busy host leaves the job queued.
|
||||
export async function createServer(input: CreateServerInput): Promise<CreateServerResult> {
|
||||
const queued = queueCreateServer(input);
|
||||
if (!queued.ok) return queued;
|
||||
const existing = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(queued.deploymentId) as { state: string };
|
||||
if (existing.state !== 'queued') return queued;
|
||||
const outcomes = await creationPass();
|
||||
const outcome = outcomes.get(queued.deploymentId);
|
||||
if (outcome instanceof Error) throw outcome;
|
||||
return outcome ?? queued;
|
||||
}
|
||||
|
||||
function prepareCreation(input: CreateServerInput, releasingServers: string[] = []): PreparedCreation | Extract<CreateServerResult, { ok: false }> {
|
||||
const game = input.game ?? 't5';
|
||||
if (game !== 't5' && game !== 't6') {
|
||||
return { ok: false, kind: 'bad-request', message: 'Supported games are t5 and t6' };
|
||||
}
|
||||
if (input.gameMode !== 'zombie') {
|
||||
return { ok: false, kind: 'unknown-map', message: `unsupported game mode: ${input.gameMode}` };
|
||||
}
|
||||
if (!config.komodo.address || !config.komodo.apiKey || !config.komodo.apiSecret) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'KOMODO_ADDRESS / KOMODO_API_KEY / KOMODO_API_SECRET must be set' };
|
||||
}
|
||||
if (!config.komodo.targetServer) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'KOMODO_TARGET_SERVER must name the Komodo host that runs new deployments' };
|
||||
}
|
||||
|
||||
const mapEntry = zombieMapBySlug(game, input.mapSlug);
|
||||
if (!mapEntry) {
|
||||
return { ok: false, kind: 'unknown-map', message: `no ${game} zombie map with slug '${input.mapSlug}'` };
|
||||
}
|
||||
|
||||
const baseName = `${game}_${input.gameMode}_${mapEntry.slug}`;
|
||||
const lobby = nextLobbyNumber(baseName);
|
||||
if (lobby === null) {
|
||||
return { ok: false, kind: 'no-slot', message: `all lobby slots (1..20) for ${baseName} are taken` };
|
||||
}
|
||||
const name = `${baseName}_${lobby}`;
|
||||
if (input.expectedName !== undefined && input.expectedName !== name) {
|
||||
return { ok: false, kind: 'no-slot', message: 'Available lobby changed. Review the updated server ID and key, then deploy again.' };
|
||||
}
|
||||
const plutoKey = input.plutoKey?.trim() || getServerKey(name);
|
||||
if (!plutoKey) return { ok: false, kind: 'bad-request', message: `No saved key for ${name}. Enter a Plutonium server key.` };
|
||||
|
||||
const port = nextAvailablePort([], releasingServers);
|
||||
if (port === null) {
|
||||
return {
|
||||
ok: false,
|
||||
kind: 'no-port',
|
||||
message: `no free port in ${config.portRange.min}..${config.portRange.max} — destroy an existing server first`
|
||||
};
|
||||
}
|
||||
|
||||
// Save before external operations so a failed deployment can also be retried.
|
||||
try { saveServerKey(name, plutoKey); }
|
||||
catch (err) { return { ok: false, kind: 'bad-request', message: (err as Error).message }; }
|
||||
|
||||
return { ok: true, name, port, plutoKey, mapEntry };
|
||||
}
|
||||
|
||||
async function createServerUnlocked(input: CreateServerInput, prepared: PreparedCreation, operation: string): Promise<CreateServerResult> {
|
||||
const { name, port, plutoKey, mapEntry } = prepared;
|
||||
const game = gameOfServer(name);
|
||||
// Resolve once more at provisioning so promotions made while queued also apply.
|
||||
// Persist the exact attempted files for history, never as input to future launches.
|
||||
const recipe = captureRecipe(game, mapEntry.slug);
|
||||
db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)
|
||||
ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`)
|
||||
.run(operation, JSON.stringify(recipe));
|
||||
const configFile = recipe.configFile;
|
||||
deploymentStage(operation, 'Preparing server files');
|
||||
const hostDir = serverDir(name);
|
||||
|
||||
// Step 1: filesystem. mkdir mode is masked by umask, so chmod explicitly
|
||||
// after — plutainer expects to be able to write to /home/plutainer/app.
|
||||
try {
|
||||
await mkdir(hostDir, { recursive: true, mode: 0o777 });
|
||||
await chmod(hostDir, 0o777);
|
||||
for (const file of recipe.files) {
|
||||
const dest = destOfKind(name, file.kind);
|
||||
const target = join(dest, file.path);
|
||||
await mkdir(dirname(target), { recursive: true, mode: 0o777 });
|
||||
// Make every newly created ancestor writable by Plutainer, even
|
||||
// when the manager's umask removed write permissions at mkdir.
|
||||
for (let dir = dirname(target); dir !== join(hostDir) && dir !== dirname(dir); dir = dirname(dir)) {
|
||||
await chmod(dir, 0o777);
|
||||
}
|
||||
await writeFile(target, Buffer.from(file.content, 'base64'));
|
||||
await chmod(target, 0o666);
|
||||
}
|
||||
} catch (err) {
|
||||
return { ok: false, kind: 'fs', message: `failed to create ${hostDir}: ${(err as Error).message}` };
|
||||
}
|
||||
|
||||
// Step 2: Komodo CreateDeployment.
|
||||
deploymentStage(operation, 'Creating Komodo deployment');
|
||||
const spec = buildSpec({ game, name, port, hostDir, plutoKey, recipe });
|
||||
const createRes = await komodoCreateDeployment(spec);
|
||||
if (!createRes.ok) {
|
||||
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
|
||||
return { ok: false, kind: 'komodo', message: `CreateDeployment: ${createRes.message}` };
|
||||
}
|
||||
|
||||
// Step 3: DB row — insert *before* execute/Deploy so the sweep sees the
|
||||
// server immediately even if Deploy hangs.
|
||||
try {
|
||||
db().transaction(() => {
|
||||
db().prepare(
|
||||
`INSERT INTO servers
|
||||
(name, komodo_type, komodo_name, game, map,
|
||||
game_mode, map_slug, host_dir, config_file, game_id, image,
|
||||
game_port, created_by, container_state)
|
||||
VALUES
|
||||
(?, 'deployment', ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, 'starting')`
|
||||
).run(
|
||||
name, name, game, mapEntry.canonical,
|
||||
input.gameMode, mapEntry.slug, hostDir,
|
||||
configFile, game === 't6' ? 't6zm' : 't5sp', recipe.image,
|
||||
port, input.createdBy ?? 'ui'
|
||||
);
|
||||
for (const file of recipe.files) {
|
||||
const versionId = file.versionId && getVersion(file.versionId) ? file.versionId : null;
|
||||
recordScript(name, file.path, versionId, file.sha256, file.size, file.path === MAP_CONFIG_FILE ? 'create-map' : 'create');
|
||||
}
|
||||
})();
|
||||
} catch (err) {
|
||||
// Roll back Komodo + FS if the DB insert fails (very unlikely — most
|
||||
// often a race where the same name got inserted twice).
|
||||
await komodoDestroyDeployment(name).catch(() => {});
|
||||
await komodoDeleteDeployment(name).catch(() => {});
|
||||
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
|
||||
return { ok: false, kind: 'db', message: `failed to insert server row: ${(err as Error).message}` };
|
||||
}
|
||||
|
||||
// Step 4: apply Komodo tags so operators can filter the deployment list
|
||||
// by game/map. Best-effort — a tag failure doesn't roll back the server;
|
||||
// the deployment is fully functional without cosmetic tags. Fires in
|
||||
// parallel to keep the create latency down.
|
||||
void applyDeploymentTags(name, game, mapEntry.canonical);
|
||||
|
||||
// Step 5: execute/Deploy — actually start the container.
|
||||
deploymentStage(operation, 'Starting container');
|
||||
const deployRes = await komodoDeploy(name);
|
||||
if (!deployRes.ok) {
|
||||
if (deployRes.kind === 'unreachable' || (deployRes.httpStatus !== undefined && deployRes.httpStatus >= 500)) {
|
||||
// A timeout or server error may arrive after the remote boot began.
|
||||
// Preserve files, identity, and capacity until readiness is known.
|
||||
recordDeploymentEvent(operation, 'Startup response uncertain; verifying the existing server: ' + deployRes.message);
|
||||
return { ok: true, name, port, deploymentId: operation };
|
||||
}
|
||||
// Container never started — undo everything so retrying is clean.
|
||||
await komodoDeleteDeployment(name).catch(() => {});
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
|
||||
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
|
||||
return { ok: false, kind: 'komodo', message: `Deploy: ${deployRes.message}` };
|
||||
}
|
||||
|
||||
return { ok: true, name, port, deploymentId: operation };
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure `game` and `map` tags exist in Komodo, then attach both to the given
|
||||
* deployment. Purely cosmetic — any failure here logs and returns; the caller
|
||||
* treats it as fire-and-forget.
|
||||
*/
|
||||
async function applyDeploymentTags(deployment: string, gameTagName: string, mapTagName: string): Promise<void> {
|
||||
try {
|
||||
const [gameTag, mapTag] = await Promise.all([
|
||||
komodoEnsureTag(gameTagName),
|
||||
komodoEnsureTag(mapTagName)
|
||||
]);
|
||||
const tagIds: string[] = [];
|
||||
if (gameTag.ok) tagIds.push(gameTag.id);
|
||||
else console.warn(`[createServer] tag "${gameTagName}": ${gameTag.message}`);
|
||||
if (mapTag.ok) tagIds.push(mapTag.id);
|
||||
else console.warn(`[createServer] tag "${mapTagName}": ${mapTag.message}`);
|
||||
if (tagIds.length === 0) return;
|
||||
|
||||
const res = await komodoUpdateTagsOnResource('Deployment', deployment, tagIds);
|
||||
if (!res.ok) {
|
||||
console.warn(`[createServer] UpdateResourceMeta (tags) for ${deployment}: ${res.message}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`[createServer] tag application for ${deployment} threw:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
const destroying = new Set<string>();
|
||||
|
||||
// Persist the request before returning so navigation and client disconnects cannot lose it.
|
||||
function hasActiveCreation(name: string): boolean {
|
||||
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name);
|
||||
}
|
||||
|
||||
export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
|
||||
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action IN ('start','restart','stop','rebuild')
|
||||
AND state IN ('waiting','queued','running')`).get(name)) {
|
||||
return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] };
|
||||
}
|
||||
const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
|
||||
if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
|
||||
const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
|
||||
if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id };
|
||||
if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] };
|
||||
const operation = db().transaction(() => {
|
||||
const id = beginDeployment(name, 'destroy', triggeredBy);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting to verify the server is empty');
|
||||
return id;
|
||||
})();
|
||||
return { ok: true, warnings: [], deploymentId: operation };
|
||||
}
|
||||
|
||||
let destructionTick: Promise<void> | null = null;
|
||||
export function processDestructions(): Promise<void> {
|
||||
if (destructionTick) return destructionTick;
|
||||
destructionTick = runDestructions().finally(() => { destructionTick = null; });
|
||||
return destructionTick;
|
||||
}
|
||||
|
||||
async function runDestructions(): Promise<void> {
|
||||
const queued = db().prepare("SELECT id, server, stage FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string; stage: string }[];
|
||||
for (const operation of queued) {
|
||||
if (creating || destroying.has(operation.server)) continue;
|
||||
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
|
||||
destroying.add(operation.server);
|
||||
try {
|
||||
if (operation.stage === REMOVING_SERVER) {
|
||||
const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create'
|
||||
AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id);
|
||||
if (newer) {
|
||||
finishDeployment(operation.id, { ok: false, message: 'A newer creation exists; local cleanup cannot safely resume.' });
|
||||
continue;
|
||||
}
|
||||
if (!claimRemoval(operation.id)) continue;
|
||||
finishDeployment(operation.id, await cleanupDestroyedServer(operation.server, operation.id));
|
||||
continue;
|
||||
}
|
||||
if (!getServer(operation.server)) {
|
||||
if (hasActiveCreation(operation.server)) {
|
||||
deploymentStage(operation.id, 'Waiting for server creation to finish');
|
||||
continue;
|
||||
}
|
||||
finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' });
|
||||
continue;
|
||||
}
|
||||
// A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
|
||||
let players = await rconGetPlayerCount(operation.server);
|
||||
if (players === null) {
|
||||
const health = await komodoInspectHealthy(operation.server);
|
||||
if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
|
||||
}
|
||||
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
|
||||
if (players !== 0) {
|
||||
deploymentStage(operation.id, players === null
|
||||
? 'Waiting for player count: server could not be reached'
|
||||
: 'Waiting for server to empty: ' + players + ' player(s) online');
|
||||
continue;
|
||||
}
|
||||
if (!claimRemoval(operation.id)) continue;
|
||||
deploymentStage(operation.id, 'Server is empty (0 players verified)');
|
||||
const result = await destroyServerUnlocked(operation.server, operation.id);
|
||||
finishDeployment(operation.id, result);
|
||||
} catch (err) {
|
||||
const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string };
|
||||
if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message);
|
||||
else if (row.state !== 'cancelled') finishDeployment(operation.id, { ok: false, message: (err as Error).message });
|
||||
} finally { destroying.delete(operation.server); }
|
||||
}
|
||||
}
|
||||
|
||||
async function destroyServerUnlocked(name: string, operation: string): Promise<DestroyServerResult> {
|
||||
const row = getServer(name);
|
||||
if (!row) {
|
||||
return { ok: false, kind: 'not-found', message: `no server named '${name}'`, warnings: [] };
|
||||
}
|
||||
|
||||
const warnings: string[] = [];
|
||||
|
||||
// 1: stop + remove the container. execute/DestroyDeployment queues the
|
||||
// teardown; the response comes back before the container is actually gone.
|
||||
deploymentStage(operation, 'Stopping and removing container');
|
||||
const destroyRes = await komodoDestroyDeployment(name);
|
||||
if (!destroyRes.ok) {
|
||||
return { ok: false, kind: 'komodo', message: 'DestroyDeployment failed: ' + destroyRes.message, warnings };
|
||||
}
|
||||
|
||||
// 2: wait for the container to actually disappear. Komodo refuses to
|
||||
// DeleteDeployment while a container still exists for it, so poll the
|
||||
// health probe until it stops reporting a running state. `ok:false` here
|
||||
// is the *success* signal — it means the container is no longer inspectable.
|
||||
const pollDeadline = Date.now() + 30_000;
|
||||
while (Date.now() < pollDeadline) {
|
||||
const health = await komodoInspectHealthy(name);
|
||||
if (!health.ok) break;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
}
|
||||
|
||||
// 3: remove the deployment entity itself.
|
||||
deploymentStage(operation, 'Deleting Komodo deployment');
|
||||
const deleteRes = await komodoDeleteDeployment(name);
|
||||
if (!deleteRes.ok) {
|
||||
return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings };
|
||||
}
|
||||
|
||||
return cleanupDestroyedServer(name, operation);
|
||||
}
|
||||
|
||||
async function cleanupDestroyedServer(name: string, operation: string): Promise<DestroyServerResult> {
|
||||
const row = getServer(name);
|
||||
const warnings: string[] = [];
|
||||
// Persist the checkpoint before local work so failed cleanup can resume without RCON.
|
||||
// Legacy operations already recorded this same stage after Komodo deletion.
|
||||
deploymentStage(operation, REMOVING_SERVER);
|
||||
// Host directory. Fall back to derived path if host_dir wasn't set on
|
||||
// legacy rows.
|
||||
const hostDir = row?.host_dir ?? serverDir(name);
|
||||
try {
|
||||
await rm(hostDir, { recursive: true, force: true });
|
||||
} catch (err) {
|
||||
warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
// Remove server-owned beta and deployment rows before the registry row.
|
||||
// Test evidence cascades from runs; inventory and snapshots cascade from servers.
|
||||
try {
|
||||
const tx = db().transaction((n: string) => {
|
||||
db().prepare('DELETE FROM deployment_servers WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM deployment_files WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_servers WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_server_scripts WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_test_runs WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_test_coverage WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_file_changes WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(n);
|
||||
});
|
||||
tx(name);
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
kind: 'db',
|
||||
message: `failed to remove DB rows for ${name}: ${(err as Error).message}`,
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
const unfinished = db().prepare("SELECT id, action FROM server_deployments WHERE server = ? AND state = 'awaiting-health'").all(name) as { id: string; action: string }[];
|
||||
for (const { id, action } of unfinished) finishDeployment(id, { ok: false, message: action === 'create'
|
||||
? 'Server destroyed before creation health checks completed' : 'Server destroyed before operation verification completed' });
|
||||
return { ok: true, warnings };
|
||||
}
|
||||
interface CreationJob {
|
||||
deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
|
||||
}
|
||||
|
||||
async function provisionCreation(id: string): Promise<CreateServerResult> {
|
||||
const job = db().prepare('SELECT j.*, d.triggered_by FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE j.deployment_id = ?').get(id) as CreationJob;
|
||||
creating = true;
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(id);
|
||||
try {
|
||||
const mapEntry = zombieMapBySlug(gameOfServer(job.server), job.map_slug);
|
||||
const plutoKey = getServerKey(job.server);
|
||||
if (!mapEntry || !plutoKey) {
|
||||
const result = { ok: false as const, kind: 'bad-request' as const, message: 'Saved server key or selected map is no longer available' };
|
||||
finishDeployment(id, result);
|
||||
return result;
|
||||
}
|
||||
const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by },
|
||||
{ ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
|
||||
if (!result.ok) finishDeployment(id, result);
|
||||
else {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for Komodo health');
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
const server = getServer(job.server);
|
||||
if (server?.game_port === job.port) {
|
||||
// A lost response may have happened after Komodo started the server.
|
||||
// Keep its slot and observe readiness instead of starting another boot.
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message);
|
||||
deploymentStage(id, 'Verifying existing server after interrupted startup');
|
||||
} else finishDeployment(id, { ok: false, message: (err as Error).message });
|
||||
throw err;
|
||||
} finally { creating = false; }
|
||||
}
|
||||
|
||||
type CreationOutcomes = Map<string, CreateServerResult | Error>;
|
||||
let creationTick: Promise<CreationOutcomes> | null = null;
|
||||
function creationPass(): Promise<CreationOutcomes> {
|
||||
if (creationTick) return creationTick;
|
||||
creationTick = runCreations().finally(() => { creationTick = null; });
|
||||
return creationTick;
|
||||
}
|
||||
|
||||
export async function processCreations(): Promise<void> {
|
||||
await creationPass();
|
||||
}
|
||||
|
||||
/** One host-wide FIFO, shared by collections and individual launches. */
|
||||
export function creationQueue() {
|
||||
type Entry = { id: string; server: string; state: string; stage: string };
|
||||
const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments
|
||||
WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
|
||||
ORDER BY started_at, rowid`).all() as Entry[];
|
||||
return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
|
||||
}
|
||||
|
||||
function creationBlocker(id: string): string | null {
|
||||
const dependency = db().prepare(`SELECT d.server, d.state FROM server_creation_dependencies dep
|
||||
JOIN server_deployments d ON d.id = dep.removal_id
|
||||
WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { server: string; state: string } | undefined;
|
||||
if (dependency) return `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`;
|
||||
const occupied = db().prepare(`SELECT s.name FROM servers s JOIN server_creation_jobs j ON s.game_port = j.port
|
||||
WHERE j.deployment_id = ? LIMIT 1`).get(id) as { name: string } | undefined;
|
||||
return occupied ? `Waiting for port to be released by ${occupied.name}` : null;
|
||||
}
|
||||
|
||||
async function runCreations(): Promise<CreationOutcomes> {
|
||||
const outcomes: CreationOutcomes = new Map();
|
||||
if (creating || destroying.size > 0) return outcomes;
|
||||
// Restart recovery is observation-only: never repeat external creation or
|
||||
// deployment. A known interrupted startup must retain its readiness gate.
|
||||
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
|
||||
for (const { id } of interrupted) {
|
||||
if (resumeVerification(id)) continue;
|
||||
// A queued removal does not make the interrupted container disappear.
|
||||
// Keep its slot until removal finishes, without resuming normal work or
|
||||
// confusing it with a newer creation of the same server identity.
|
||||
const removing = db().prepare(`SELECT 1 FROM server_deployments d
|
||||
JOIN server_creation_jobs j ON j.deployment_id = d.id
|
||||
JOIN servers s ON s.name = j.server AND s.game_port = j.port
|
||||
WHERE d.id = ? AND EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server
|
||||
AND newer.rowid > d.rowid AND newer.action = 'destroy' AND newer.state IN ('waiting', 'running'))
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server
|
||||
AND newer.rowid > d.rowid AND newer.action = 'create')`).get(id);
|
||||
if (removing) {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for requested server removal after interrupted startup');
|
||||
}
|
||||
}
|
||||
|
||||
// Check only previously started servers. Newly launched servers keep their
|
||||
// slot until a later pass, even if a health probe would answer immediately.
|
||||
const checking = db().prepare(`SELECT j.* FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[];
|
||||
for (const job of checking) {
|
||||
const id = job.deployment_id;
|
||||
if (getServer(job.server)?.game_port !== job.port) {
|
||||
finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' });
|
||||
continue;
|
||||
}
|
||||
if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
|
||||
try {
|
||||
const komodo = await komodoInspectHealthy(job.server);
|
||||
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
|
||||
const rcon = await rconStatusResult(job.server);
|
||||
if (!rcon.ok) { deploymentStage(id, 'Waiting for RCON health: ' + rcon.reason); continue; }
|
||||
const actualMap = rcon.status.map;
|
||||
if (actualMap !== job.expected_map) {
|
||||
deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', RCON reports ' + (actualMap ?? 'no map'));
|
||||
continue;
|
||||
}
|
||||
deploymentStage(id, 'Verified Komodo healthy, RCON healthy, and map ' + actualMap);
|
||||
finishDeployment(id, { ok: true });
|
||||
} catch (err) {
|
||||
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot the budget once. Fast success/failure must not turn a single
|
||||
// worker pass into an unbounded stream of expensive container startups.
|
||||
const queue = creationQueue();
|
||||
const available = Math.max(0, config.createConcurrency - queue.active.length);
|
||||
const eligible = queue.queued.filter(({ id }) => {
|
||||
const blocker = creationBlocker(id);
|
||||
if (blocker) deploymentStage(id, blocker);
|
||||
return !blocker;
|
||||
});
|
||||
for (const { id } of eligible.slice(0, available)) {
|
||||
if (creating || destroying.size > 0) break;
|
||||
// API calls can cancel a selected job or resume an older startup while
|
||||
// a previous provision awaits Komodo. Recheck without expanding this pass.
|
||||
if (creationQueue().active.length >= config.createConcurrency) break;
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
|
||||
const blocker = creationBlocker(id);
|
||||
if (blocker) {
|
||||
deploymentStage(id, blocker);
|
||||
continue;
|
||||
}
|
||||
try { outcomes.set(id, await provisionCreation(id)); }
|
||||
catch (err) {
|
||||
const failure = err instanceof Error ? err : new Error(String(err));
|
||||
outcomes.set(id, failure);
|
||||
console.error('[creation] provisioning failed:', failure);
|
||||
}
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
Reference in New Issue
Block a user