refactor: make provisioning own instance lifecycle operations
This commit is contained in:
@@ -24,6 +24,7 @@ export interface MaintenanceJob {
|
||||
target_revision: number | null;
|
||||
requested_at: string;
|
||||
restarted_at: string | null;
|
||||
provisioning_id: string | null;
|
||||
finished_at: string | null;
|
||||
detail: string | null;
|
||||
}
|
||||
|
||||
@@ -261,6 +261,14 @@ export function db(): Database.Database {
|
||||
handle.exec(schemaSql);
|
||||
migrateProvisioningActions(handle);
|
||||
migrateTable(handle, 'restart_rollout', [{ name: 'operation_id', ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' }]);
|
||||
migrateTable(handle, 'server_deployments', [
|
||||
{ name: 'submitted_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN submitted_at TEXT' },
|
||||
{ name: 'checked_at', ddl: 'ALTER TABLE server_deployments ADD COLUMN checked_at TEXT' },
|
||||
{ name: 'players', ddl: 'ALTER TABLE server_deployments ADD COLUMN players INTEGER' }
|
||||
]);
|
||||
for (const table of ['deployment_servers', 'beta_servers', 'maintenance_jobs']) {
|
||||
migrateTable(handle, table, [{ name: 'provisioning_id', ddl: `ALTER TABLE ${table} ADD COLUMN provisioning_id TEXT` }]);
|
||||
}
|
||||
migrateServers(handle);
|
||||
// Column additions must run BEFORE any statement that references the
|
||||
// new columns. schema.sql cannot include those references — see the
|
||||
|
||||
@@ -441,6 +441,16 @@ CREATE TABLE IF NOT EXISTS server_rebuild_jobs (
|
||||
removed_at TEXT
|
||||
);
|
||||
|
||||
-- Stable links from consumers to lifecycle jobs. Sources survive their caller's
|
||||
-- history retention, while current pointers live on the caller's server rows.
|
||||
CREATE TABLE IF NOT EXISTS provisioning_sources (
|
||||
operation_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('campaign','beta','maintenance')),
|
||||
source_id TEXT NOT NULL,
|
||||
server TEXT NOT NULL,
|
||||
PRIMARY KEY (operation_id, kind, source_id, server)
|
||||
);
|
||||
|
||||
-- Blueprints retain map/count selections only. The legacy column name remains
|
||||
-- compatible with older snapshot-based definitions, read as selections at runtime.
|
||||
CREATE TABLE IF NOT EXISTS server_blueprints (
|
||||
|
||||
@@ -154,6 +154,7 @@ export interface BetaServerRow {
|
||||
error: string | null;
|
||||
deployed_at: string | null;
|
||||
restarted_at: string | null;
|
||||
provisioning_id: string | null;
|
||||
healthy_at: string | null;
|
||||
match_seen_at: string | null;
|
||||
updated_at: string;
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSum
|
||||
import { db } from '../db/index.js';
|
||||
import { config, komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js';
|
||||
import { creationQueue, destroyServer, nextAvailablePort, nextLobbyNumber, queueCreateServer } from './create.js';
|
||||
import { creationQueue, destroyServer, nextAvailablePort, nextLobbyNumber, queueCreateServer } from '../provisioning/index.js';
|
||||
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
|
||||
import { housePartyTargets } from './house-party.js';
|
||||
import { getServerKey } from './server-keys.js';
|
||||
|
||||
@@ -1,801 +1 @@
|
||||
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
|
||||
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './deployments.js';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { LaunchPreview } from '$lib/deployment-plan.js';
|
||||
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from './launch-recipes.js';
|
||||
import { mkdir, chmod, rm, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { getServerKey, saveServerKey } from './server-keys.js';
|
||||
import { getServer } from './servers.js';
|
||||
import { zombieMapBySlug, type GameMode, type ZombieMapEntry, type ZombieGame } from './map-aliases.js';
|
||||
import {
|
||||
komodoCreateDeployment,
|
||||
komodoDeploy,
|
||||
komodoDestroyDeployment,
|
||||
komodoDeleteDeployment,
|
||||
komodoInspectHealthy,
|
||||
komodoEnsureTag,
|
||||
komodoUpdateTagsOnResource,
|
||||
type CreateDeploymentSpec,
|
||||
type DeploymentEnvVar
|
||||
} from '../integrations/komodo.js';
|
||||
import { kick } from '../pipeline/worker.js';
|
||||
import { MAP_CONFIG_FILE } from './server-config.js';
|
||||
import { recordScript } from './inventory.js';
|
||||
import { getVersion } from './scripts.js';
|
||||
|
||||
export type CreateServerFailure =
|
||||
| 'bad-request'
|
||||
| 'unconfigured' // Komodo creds / target server missing
|
||||
| 'unknown-map' // slug isn't in the catalog
|
||||
| 'no-slot' // all lobby slots for this map are taken
|
||||
| 'no-port' // port range exhausted
|
||||
| 'fs' // couldn't create host dir
|
||||
| 'komodo' // Komodo CreateDeployment / Deploy rejected
|
||||
| 'db'; // shouldn't happen, but wraps INSERT failures
|
||||
|
||||
export type CreateServerResult =
|
||||
| { ok: true; name: string; port: number; deploymentId: string }
|
||||
| { ok: false; kind: CreateServerFailure; message: string };
|
||||
|
||||
export type DestroyServerResult =
|
||||
| { ok: true; warnings: string[] }
|
||||
| { ok: false; kind: 'not-found' | 'conflict' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] };
|
||||
|
||||
export interface CreateServerInput {
|
||||
game?: ZombieGame; // omitted by older clients: T5
|
||||
gameMode: GameMode; // 'zombie' — only value supported today
|
||||
mapSlug: string; // 'cosmodrome'
|
||||
plutoKey?: string; // saved for this server ID when supplied
|
||||
expectedName?: string; // optional preview guard; never selects a lobby
|
||||
previewId?: string;
|
||||
createdBy?: string; // 'ui' by default
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the lowest lobby number `n` such that `t5_<mode>_<slug>_<n>` isn't in
|
||||
* the servers table. Caps at 20 to stay inside the port range — with 20 total
|
||||
* ports across the whole fleet, no one map can hold more than that.
|
||||
*/
|
||||
export function nextLobbyNumber(baseName: string, additionalNames: string[] = []): number | null {
|
||||
const rows = db()
|
||||
.prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
|
||||
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
|
||||
.all(`${baseName}_%`) as Array<{ name: string }>;
|
||||
const taken = new Set<number>();
|
||||
const prefix = `${baseName}_`;
|
||||
for (const r of [...rows, ...additionalNames.map(name => ({ name }))]) {
|
||||
if (!r.name.startsWith(prefix)) continue;
|
||||
const tail = r.name.slice(prefix.length);
|
||||
if (/^\d+$/.test(tail)) taken.add(Number.parseInt(tail, 10));
|
||||
}
|
||||
for (let n = 1; n <= 20; n++) {
|
||||
if (!taken.has(n)) return n;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lowest unused port in the configured range. Reads every server's game_port
|
||||
* from the DB (which EC controls end-to-end under the new naming scheme, so
|
||||
* the DB is authoritative — no need to ask Komodo).
|
||||
*/
|
||||
export function nextAvailablePort(additionalPorts: number[] = [], releasingServers: string[] = []): number | null {
|
||||
const rows = db()
|
||||
.prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
|
||||
SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
|
||||
.all() as Array<{ name: string; game_port: number }>;
|
||||
const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
|
||||
for (let p = config.portRange.min; p <= config.portRange.max; p++) {
|
||||
if (!taken.has(p)) return p;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function buildEnv(args: {
|
||||
game: ZombieGame;
|
||||
name: string;
|
||||
port: number;
|
||||
plutoKey: string;
|
||||
}): DeploymentEnvVar[] {
|
||||
return [
|
||||
{ variable: 'PLUTAINER_SERVER_NAME', value: args.name },
|
||||
{ variable: 'PLUTAINER_PORT', value: String(args.port) },
|
||||
{ variable: 'PLUTAINER_GAME', value: args.game === 't6' ? 't6zm' : 't5sp' },
|
||||
{ variable: 'PLUTAINER_CONFIG_FILE', value: configFileForGame(args.game) },
|
||||
// Plutainer seeds/loads the main config first. Load our generated map
|
||||
// config afterward, then rotate. A separate exec avoids quoted values
|
||||
// in EXTRA_ARGS, which Plutainer splits on whitespace.
|
||||
{ variable: 'PLUTAINER_MAP_ROTATE', value: 'false' },
|
||||
// Give telemetry scripts the registry name before the first map starts.
|
||||
{ variable: 'PLUTAINER_EXTRA_ARGS', value: `+exec ${MAP_CONFIG_FILE} +set tws_server_id ${args.name} +map_rotate` },
|
||||
{ variable: 'PLUTO_SERVER_KEY', value: args.plutoKey },
|
||||
{ variable: 'PLUTAINER_RCON_PASSWORD', value: config.rcon.password }
|
||||
];
|
||||
}
|
||||
|
||||
function buildSpec(args: {
|
||||
game: ZombieGame;
|
||||
name: string;
|
||||
port: number;
|
||||
hostDir: string;
|
||||
plutoKey: string;
|
||||
recipe: LaunchRecipe;
|
||||
}): CreateDeploymentSpec {
|
||||
return {
|
||||
name: args.name,
|
||||
server: config.komodo.targetServer,
|
||||
image: args.recipe.image,
|
||||
network: config.komodo.defaultNetwork,
|
||||
// Container listens on PLUTAINER_PORT (== args.port), so host and
|
||||
// container ports match — no NAT inside the bridge network.
|
||||
ports: [
|
||||
`${args.port}:${args.port}/udp`,
|
||||
`${args.port}:${args.port}/tcp`
|
||||
],
|
||||
volumes: [
|
||||
`${args.hostDir}:/home/plutainer/app`,
|
||||
`${args.game === 't6' ? config.komodo.t6GameFilesVolume : config.komodo.gameFilesVolume}:/home/plutainer/gamefiles:ro`
|
||||
],
|
||||
environment: buildEnv(args)
|
||||
};
|
||||
}
|
||||
|
||||
let creating = false;
|
||||
interface PreparedCreation {
|
||||
ok: true; name: string; port: number; plutoKey: string; mapEntry: ZombieMapEntry;
|
||||
}
|
||||
|
||||
interface PreviewRow {
|
||||
id: string; expires_at: string; server: string | null; port: number | null;
|
||||
recipe: string; blueprint_name: string | null; deployment_id: string | null;
|
||||
}
|
||||
|
||||
/** Read-only checks; a preview never reserves a slot or writes credentials. */
|
||||
export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown }): LaunchPreview {
|
||||
if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.');
|
||||
let recipe: LaunchRecipe;
|
||||
let blueprintName: string | null = null;
|
||||
if (input.blueprintId !== undefined) {
|
||||
if (typeof input.blueprintId !== 'string') throw new Error('Invalid blueprint.');
|
||||
({ recipe, blueprintName } = blueprintRecipe(input.blueprintId));
|
||||
} else if (input.deploymentId !== undefined) {
|
||||
if (typeof input.deploymentId !== 'string') throw new Error('Invalid deployment.');
|
||||
const source = recipeForDeployment(input.deploymentId);
|
||||
if (!source) throw new Error('This deployment has no recorded launch recipe.');
|
||||
blueprintName = source.blueprintName;
|
||||
// History describes the previous launch; only its map selection is reusable.
|
||||
recipe = captureRecipe(source.recipe.game, source.recipe.mapSlug);
|
||||
} else {
|
||||
if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.');
|
||||
recipe = captureRecipe(input.game, input.mapSlug);
|
||||
}
|
||||
const lobby = nextLobbyNumber(`${recipe.game}_zombie_${recipe.mapSlug}`);
|
||||
const name = lobby === null ? null : `${recipe.game}_zombie_${recipe.mapSlug}_${lobby}`;
|
||||
const port = nextAvailablePort();
|
||||
const configured = !!(config.komodo.address && config.komodo.apiKey && config.komodo.apiSecret && config.komodo.targetServer);
|
||||
const checks: LaunchPreview['checks'] = [
|
||||
{ label: 'Komodo settings', state: configured ? 'pass' : 'blocked', detail: configured ? 'Connection settings and target host are configured.' : 'Configure Komodo credentials and a target host before launching.' },
|
||||
{ label: 'Health verification', state: config.rcon.password ? 'pass' : 'blocked', detail: config.rcon.password ? 'RCON is configured. Container, RCON, and map checks run after launch.' : 'Configure RCON_PASSWORD so the new server can be verified.' },
|
||||
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All 20 lobbies for this map are occupied or reserved.' },
|
||||
{ label: 'Game port', state: port === null ? 'blocked' : 'pass', detail: port === null ? 'No free game port. Free capacity before launching.' : `${port} / UDP and TCP. Availability is checked again at launch.` },
|
||||
{ label: 'Promoted configuration', state: 'pass', detail: recipe.configSource },
|
||||
{ label: 'Game files and host readiness', state: 'info', detail: 'The host must have the selected game and DLC files. This preview does not probe volumes, images, credentials, or network access.' }
|
||||
];
|
||||
const id = randomUUID();
|
||||
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
||||
db().prepare('DELETE FROM server_launch_previews WHERE expires_at < ? AND deployment_id IS NULL').run(new Date().toISOString());
|
||||
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
|
||||
return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName,
|
||||
checks, ready: checks.every(check => check.state !== 'blocked') };
|
||||
}
|
||||
|
||||
// releasingServers is internal to atomic collection queueing; HTTP callers cannot set it.
|
||||
export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult {
|
||||
return db().transaction((): CreateServerResult => {
|
||||
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined;
|
||||
if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' };
|
||||
const recipe: LaunchRecipe | undefined = preview ? JSON.parse(preview.recipe) : undefined;
|
||||
if (recipe && (recipe.game !== (input.game ?? 't5') || recipe.mapSlug !== input.mapSlug || input.gameMode !== 'zombie')) {
|
||||
return { ok: false, kind: 'bad-request', message: 'The selection does not match the reviewed launch recipe.' };
|
||||
}
|
||||
// A repeated submit after a lost response returns the same operation.
|
||||
if (preview?.deployment_id) return { ok: true, name: preview.server!, port: preview.port!, deploymentId: preview.deployment_id };
|
||||
if (preview && preview.expires_at <= new Date().toISOString()) return { ok: false, kind: 'bad-request', message: 'Launch preview expired. Review a fresh preview.' };
|
||||
if (preview && (!preview.server || !preview.port || preview.server !== nextName(input) || preview.port !== nextAvailablePort([], releasingServers))) {
|
||||
return { ok: false, kind: 'no-slot', message: 'Available lobby or port changed. Refresh the preview and review the new server key.' };
|
||||
}
|
||||
if (recipe && !config.rcon.password) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'Health-check settings changed. Refresh the preview.' };
|
||||
}
|
||||
const prepared = prepareCreation(input, releasingServers);
|
||||
if (!prepared.ok) return prepared;
|
||||
// A preview records choices, not version pins. Promotions remain authoritative.
|
||||
const currentRecipe = captureRecipe(input.game ?? 't5', prepared.mapEntry.slug);
|
||||
const id = beginDeployment(prepared.name, 'create', input.createdBy);
|
||||
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, blueprint_name) VALUES (?, ?, ?)')
|
||||
.run(id, JSON.stringify(currentRecipe), preview?.blueprint_name ?? null);
|
||||
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id = ? WHERE id = ?').run(id, preview.id);
|
||||
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical);
|
||||
deploymentStage(id, 'Queued for server creation');
|
||||
return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id };
|
||||
})();
|
||||
}
|
||||
|
||||
/** Reserve the same identity and port before removing any part of the instance. */
|
||||
export function queueRebuildServer(name: string): CreateServerResult {
|
||||
const server = getServer(name);
|
||||
const game = gameOfServer(name);
|
||||
const map = server?.map_slug && zombieMapBySlug(game, server.map_slug);
|
||||
if (!server || !map || !server.game_port || server.komodo_type !== 'deployment' || server.komodo_name !== name
|
||||
|| (server.host_dir && server.host_dir !== serverDir(name))) {
|
||||
return { ok: false, kind: 'bad-request', message: 'Rebuild requires a managed server with its original map, directory, and port.' };
|
||||
}
|
||||
if (!config.komodo.targetServer || !config.rcon.password) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' };
|
||||
if (!getServerKey(name)) return { ok: false, kind: 'bad-request', message: `Save a server key for ${name} in Keys before rebuilding.` };
|
||||
// Validate that the replacement setup can be composed before destruction.
|
||||
const recipe = captureRecipe(game, map.slug);
|
||||
return db().transaction(() => {
|
||||
const id = beginDeployment(name, 'rebuild');
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
|
||||
db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
|
||||
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe));
|
||||
deploymentStage(id, 'Waiting to rebuild the empty server');
|
||||
return { ok: true as const, name, port: server.game_port!, deploymentId: id };
|
||||
})();
|
||||
}
|
||||
|
||||
/** Caller has just verified emptiness and excluded conflicting work. */
|
||||
export async function rebuildServer(id: string): Promise<void> {
|
||||
const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.id = ? AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
|
||||
if (!job) return;
|
||||
if (creating || destroying.size || creationQueue().active.length >= config.createConcurrency) {
|
||||
// The cleanup checkpoint must survive capacity waits and app restarts.
|
||||
if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding');
|
||||
return;
|
||||
}
|
||||
// Recheck prerequisites while the original instance still exists.
|
||||
if (!getServerKey(job.server)) { finishDeployment(id, { ok: false, message: 'Saved server key was removed; rebuild did not begin.' }); return; }
|
||||
try { captureRecipe(gameOfServer(job.server), job.map_slug); }
|
||||
catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); return; }
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
|
||||
destroying.add(job.server);
|
||||
try {
|
||||
const result = job.stage === REMOVING_SERVER
|
||||
? await cleanupDestroyedServer(job.server, id) : await destroyServerUnlocked(job.server, id);
|
||||
if (!result.ok || result.warnings.length) {
|
||||
finishDeployment(id, { ok: false, message: result.ok ? 'Old server files could not be completely removed.' : result.message, warnings: result.warnings });
|
||||
return;
|
||||
}
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE server_rebuild_jobs SET removed_at = ? WHERE deployment_id = ?').run(new Date().toISOString(), id);
|
||||
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Queued for server creation');
|
||||
})();
|
||||
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
|
||||
finally { destroying.delete(job.server); }
|
||||
}
|
||||
|
||||
function nextName(input: CreateServerInput): string | null {
|
||||
const base = `${input.game ?? 't5'}_${input.gameMode}_${input.mapSlug}`;
|
||||
const lobby = nextLobbyNumber(base);
|
||||
return lobby === null ? null : `${base}_${lobby}`;
|
||||
}
|
||||
|
||||
// Internal callers use the same durable queue and capacity limit as HTTP launches.
|
||||
// Await this pass's provisioning when possible; a busy host leaves the job queued.
|
||||
export async function createServer(input: CreateServerInput): Promise<CreateServerResult> {
|
||||
const queued = queueCreateServer(input);
|
||||
if (!queued.ok) return queued;
|
||||
const existing = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(queued.deploymentId) as { state: string };
|
||||
if (existing.state !== 'queued') return queued;
|
||||
const outcomes = await creationPass();
|
||||
const outcome = outcomes.get(queued.deploymentId);
|
||||
if (outcome instanceof Error) throw outcome;
|
||||
return outcome ?? queued;
|
||||
}
|
||||
|
||||
function prepareCreation(input: CreateServerInput, releasingServers: string[] = []): PreparedCreation | Extract<CreateServerResult, { ok: false }> {
|
||||
const game = input.game ?? 't5';
|
||||
if (game !== 't5' && game !== 't6') {
|
||||
return { ok: false, kind: 'bad-request', message: 'Supported games are t5 and t6' };
|
||||
}
|
||||
if (input.gameMode !== 'zombie') {
|
||||
return { ok: false, kind: 'unknown-map', message: `unsupported game mode: ${input.gameMode}` };
|
||||
}
|
||||
if (!config.komodo.address || !config.komodo.apiKey || !config.komodo.apiSecret) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'KOMODO_ADDRESS / KOMODO_API_KEY / KOMODO_API_SECRET must be set' };
|
||||
}
|
||||
if (!config.komodo.targetServer) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'KOMODO_TARGET_SERVER must name the Komodo host that runs new deployments' };
|
||||
}
|
||||
|
||||
const mapEntry = zombieMapBySlug(game, input.mapSlug);
|
||||
if (!mapEntry) {
|
||||
return { ok: false, kind: 'unknown-map', message: `no ${game} zombie map with slug '${input.mapSlug}'` };
|
||||
}
|
||||
|
||||
const baseName = `${game}_${input.gameMode}_${mapEntry.slug}`;
|
||||
const lobby = nextLobbyNumber(baseName);
|
||||
if (lobby === null) {
|
||||
return { ok: false, kind: 'no-slot', message: `all lobby slots (1..20) for ${baseName} are taken` };
|
||||
}
|
||||
const name = `${baseName}_${lobby}`;
|
||||
if (input.expectedName !== undefined && input.expectedName !== name) {
|
||||
return { ok: false, kind: 'no-slot', message: 'Available lobby changed. Review the updated server ID and key, then deploy again.' };
|
||||
}
|
||||
const plutoKey = input.plutoKey?.trim() || getServerKey(name);
|
||||
if (!plutoKey) return { ok: false, kind: 'bad-request', message: `No saved key for ${name}. Enter a Plutonium server key.` };
|
||||
|
||||
const port = nextAvailablePort([], releasingServers);
|
||||
if (port === null) {
|
||||
return {
|
||||
ok: false,
|
||||
kind: 'no-port',
|
||||
message: `no free port in ${config.portRange.min}..${config.portRange.max} — destroy an existing server first`
|
||||
};
|
||||
}
|
||||
|
||||
// Save before external operations so a failed deployment can also be retried.
|
||||
try { saveServerKey(name, plutoKey); }
|
||||
catch (err) { return { ok: false, kind: 'bad-request', message: (err as Error).message }; }
|
||||
|
||||
return { ok: true, name, port, plutoKey, mapEntry };
|
||||
}
|
||||
|
||||
async function createServerUnlocked(input: CreateServerInput, prepared: PreparedCreation, operation: string): Promise<CreateServerResult> {
|
||||
const { name, port, plutoKey, mapEntry } = prepared;
|
||||
const game = gameOfServer(name);
|
||||
// Resolve once more at provisioning so promotions made while queued also apply.
|
||||
// Persist the exact attempted files for history, never as input to future launches.
|
||||
const recipe = captureRecipe(game, mapEntry.slug);
|
||||
db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)
|
||||
ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`)
|
||||
.run(operation, JSON.stringify(recipe));
|
||||
const configFile = recipe.configFile;
|
||||
deploymentStage(operation, 'Preparing server files');
|
||||
const hostDir = serverDir(name);
|
||||
|
||||
// Step 1: filesystem. mkdir mode is masked by umask, so chmod explicitly
|
||||
// after — plutainer expects to be able to write to /home/plutainer/app.
|
||||
try {
|
||||
await mkdir(hostDir, { recursive: true, mode: 0o777 });
|
||||
await chmod(hostDir, 0o777);
|
||||
for (const file of recipe.files) {
|
||||
const dest = destOfKind(name, file.kind);
|
||||
const target = join(dest, file.path);
|
||||
await mkdir(dirname(target), { recursive: true, mode: 0o777 });
|
||||
// Make every newly created ancestor writable by Plutainer, even
|
||||
// when the manager's umask removed write permissions at mkdir.
|
||||
for (let dir = dirname(target); dir !== join(hostDir) && dir !== dirname(dir); dir = dirname(dir)) {
|
||||
await chmod(dir, 0o777);
|
||||
}
|
||||
await writeFile(target, Buffer.from(file.content, 'base64'));
|
||||
await chmod(target, 0o666);
|
||||
}
|
||||
} catch (err) {
|
||||
return { ok: false, kind: 'fs', message: `failed to create ${hostDir}: ${(err as Error).message}` };
|
||||
}
|
||||
|
||||
// Step 2: Komodo CreateDeployment.
|
||||
deploymentStage(operation, 'Creating Komodo deployment');
|
||||
const spec = buildSpec({ game, name, port, hostDir, plutoKey, recipe });
|
||||
const createRes = await komodoCreateDeployment(spec);
|
||||
if (!createRes.ok) {
|
||||
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
|
||||
return { ok: false, kind: 'komodo', message: `CreateDeployment: ${createRes.message}` };
|
||||
}
|
||||
|
||||
// Step 3: DB row — insert *before* execute/Deploy so the sweep sees the
|
||||
// server immediately even if Deploy hangs.
|
||||
try {
|
||||
db().transaction(() => {
|
||||
db().prepare(
|
||||
`INSERT INTO servers
|
||||
(name, komodo_type, komodo_name, game, map,
|
||||
game_mode, map_slug, host_dir, config_file, game_id, image,
|
||||
game_port, created_by, container_state)
|
||||
VALUES
|
||||
(?, 'deployment', ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, 'starting')`
|
||||
).run(
|
||||
name, name, game, mapEntry.canonical,
|
||||
input.gameMode, mapEntry.slug, hostDir,
|
||||
configFile, game === 't6' ? 't6zm' : 't5sp', recipe.image,
|
||||
port, input.createdBy ?? 'ui'
|
||||
);
|
||||
for (const file of recipe.files) {
|
||||
const versionId = file.versionId && getVersion(file.versionId) ? file.versionId : null;
|
||||
recordScript(name, file.path, versionId, file.sha256, file.size, file.path === MAP_CONFIG_FILE ? 'create-map' : 'create');
|
||||
}
|
||||
})();
|
||||
} catch (err) {
|
||||
// Roll back Komodo + FS if the DB insert fails (very unlikely — most
|
||||
// often a race where the same name got inserted twice).
|
||||
await komodoDestroyDeployment(name).catch(() => {});
|
||||
await komodoDeleteDeployment(name).catch(() => {});
|
||||
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
|
||||
return { ok: false, kind: 'db', message: `failed to insert server row: ${(err as Error).message}` };
|
||||
}
|
||||
|
||||
// Step 4: apply Komodo tags so operators can filter the deployment list
|
||||
// by game/map. Best-effort — a tag failure doesn't roll back the server;
|
||||
// the deployment is fully functional without cosmetic tags. Fires in
|
||||
// parallel to keep the create latency down.
|
||||
void applyDeploymentTags(name, game, mapEntry.canonical);
|
||||
|
||||
// Step 5: execute/Deploy — actually start the container.
|
||||
deploymentStage(operation, 'Starting container');
|
||||
const deployRes = await komodoDeploy(name);
|
||||
if (!deployRes.ok) {
|
||||
if (deployRes.kind === 'unreachable' || (deployRes.httpStatus !== undefined && deployRes.httpStatus >= 500)) {
|
||||
// A timeout or server error may arrive after the remote boot began.
|
||||
// Preserve files, identity, and capacity until readiness is known.
|
||||
recordDeploymentEvent(operation, 'Startup response uncertain; verifying the existing server: ' + deployRes.message);
|
||||
return { ok: true, name, port, deploymentId: operation };
|
||||
}
|
||||
// Container never started — undo everything so retrying is clean.
|
||||
await komodoDeleteDeployment(name).catch(() => {});
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
|
||||
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
|
||||
return { ok: false, kind: 'komodo', message: `Deploy: ${deployRes.message}` };
|
||||
}
|
||||
|
||||
// Step 6: kick the sweep so RCON status polling for this server starts
|
||||
// now instead of on the next 15s tick.
|
||||
void kick().catch((err) => console.error('[createServer] post-create kick failed:', err));
|
||||
|
||||
return { ok: true, name, port, deploymentId: operation };
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure `game` and `map` tags exist in Komodo, then attach both to the given
|
||||
* deployment. Purely cosmetic — any failure here logs and returns; the caller
|
||||
* treats it as fire-and-forget.
|
||||
*/
|
||||
async function applyDeploymentTags(deployment: string, gameTagName: string, mapTagName: string): Promise<void> {
|
||||
try {
|
||||
const [gameTag, mapTag] = await Promise.all([
|
||||
komodoEnsureTag(gameTagName),
|
||||
komodoEnsureTag(mapTagName)
|
||||
]);
|
||||
const tagIds: string[] = [];
|
||||
if (gameTag.ok) tagIds.push(gameTag.id);
|
||||
else console.warn(`[createServer] tag "${gameTagName}": ${gameTag.message}`);
|
||||
if (mapTag.ok) tagIds.push(mapTag.id);
|
||||
else console.warn(`[createServer] tag "${mapTagName}": ${mapTag.message}`);
|
||||
if (tagIds.length === 0) return;
|
||||
|
||||
const res = await komodoUpdateTagsOnResource('Deployment', deployment, tagIds);
|
||||
if (!res.ok) {
|
||||
console.warn(`[createServer] UpdateResourceMeta (tags) for ${deployment}: ${res.message}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`[createServer] tag application for ${deployment} threw:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
const destroying = new Set<string>();
|
||||
|
||||
// Persist the request before returning so navigation and client disconnects cannot lose it.
|
||||
function hasActiveCreation(name: string): boolean {
|
||||
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name);
|
||||
}
|
||||
|
||||
export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
|
||||
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action IN ('start','restart','stop','rebuild')
|
||||
AND state IN ('waiting','queued','running')`).get(name)) {
|
||||
return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] };
|
||||
}
|
||||
const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
|
||||
if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
|
||||
const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
|
||||
if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id };
|
||||
if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] };
|
||||
const operation = db().transaction(() => {
|
||||
const id = beginDeployment(name, 'destroy', triggeredBy);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting to verify the server is empty');
|
||||
return id;
|
||||
})();
|
||||
return { ok: true, warnings: [], deploymentId: operation };
|
||||
}
|
||||
|
||||
let destructionTick: Promise<void> | null = null;
|
||||
export function processDestructions(): Promise<void> {
|
||||
if (destructionTick) return destructionTick;
|
||||
destructionTick = runDestructions().finally(() => { destructionTick = null; });
|
||||
return destructionTick;
|
||||
}
|
||||
|
||||
async function runDestructions(): Promise<void> {
|
||||
const queued = db().prepare("SELECT id, server, stage FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string; stage: string }[];
|
||||
for (const operation of queued) {
|
||||
if (creating || destroying.has(operation.server)) continue;
|
||||
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
|
||||
destroying.add(operation.server);
|
||||
try {
|
||||
if (operation.stage === REMOVING_SERVER) {
|
||||
const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create'
|
||||
AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id);
|
||||
if (newer) {
|
||||
finishDeployment(operation.id, { ok: false, message: 'A newer creation exists; local cleanup cannot safely resume.' });
|
||||
continue;
|
||||
}
|
||||
if (!claimRemoval(operation.id)) continue;
|
||||
finishDeployment(operation.id, await cleanupDestroyedServer(operation.server, operation.id));
|
||||
continue;
|
||||
}
|
||||
if (!getServer(operation.server)) {
|
||||
if (hasActiveCreation(operation.server)) {
|
||||
deploymentStage(operation.id, 'Waiting for server creation to finish');
|
||||
continue;
|
||||
}
|
||||
finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' });
|
||||
continue;
|
||||
}
|
||||
// A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
|
||||
let players = await rconGetPlayerCount(operation.server);
|
||||
if (players === null) {
|
||||
const health = await komodoInspectHealthy(operation.server);
|
||||
if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
|
||||
}
|
||||
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
|
||||
if (players !== 0) {
|
||||
deploymentStage(operation.id, players === null
|
||||
? 'Waiting for player count: server could not be reached'
|
||||
: 'Waiting for server to empty: ' + players + ' player(s) online');
|
||||
continue;
|
||||
}
|
||||
if (!claimRemoval(operation.id)) continue;
|
||||
deploymentStage(operation.id, 'Server is empty (0 players verified)');
|
||||
const result = await destroyServerUnlocked(operation.server, operation.id);
|
||||
finishDeployment(operation.id, result);
|
||||
} catch (err) {
|
||||
const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string };
|
||||
if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message);
|
||||
else if (row.state !== 'cancelled') finishDeployment(operation.id, { ok: false, message: (err as Error).message });
|
||||
} finally { destroying.delete(operation.server); }
|
||||
}
|
||||
}
|
||||
|
||||
async function destroyServerUnlocked(name: string, operation: string): Promise<DestroyServerResult> {
|
||||
const row = getServer(name);
|
||||
if (!row) {
|
||||
return { ok: false, kind: 'not-found', message: `no server named '${name}'`, warnings: [] };
|
||||
}
|
||||
|
||||
const warnings: string[] = [];
|
||||
|
||||
// 1: stop + remove the container. execute/DestroyDeployment queues the
|
||||
// teardown; the response comes back before the container is actually gone.
|
||||
deploymentStage(operation, 'Stopping and removing container');
|
||||
const destroyRes = await komodoDestroyDeployment(name);
|
||||
if (!destroyRes.ok) {
|
||||
return { ok: false, kind: 'komodo', message: 'DestroyDeployment failed: ' + destroyRes.message, warnings };
|
||||
}
|
||||
|
||||
// 2: wait for the container to actually disappear. Komodo refuses to
|
||||
// DeleteDeployment while a container still exists for it, so poll the
|
||||
// health probe until it stops reporting a running state. `ok:false` here
|
||||
// is the *success* signal — it means the container is no longer inspectable.
|
||||
const pollDeadline = Date.now() + 30_000;
|
||||
while (Date.now() < pollDeadline) {
|
||||
const health = await komodoInspectHealthy(name);
|
||||
if (!health.ok) break;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
}
|
||||
|
||||
// 3: remove the deployment entity itself.
|
||||
deploymentStage(operation, 'Deleting Komodo deployment');
|
||||
const deleteRes = await komodoDeleteDeployment(name);
|
||||
if (!deleteRes.ok) {
|
||||
return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings };
|
||||
}
|
||||
|
||||
return cleanupDestroyedServer(name, operation);
|
||||
}
|
||||
|
||||
async function cleanupDestroyedServer(name: string, operation: string): Promise<DestroyServerResult> {
|
||||
const row = getServer(name);
|
||||
const warnings: string[] = [];
|
||||
// Persist the checkpoint before local work so failed cleanup can resume without RCON.
|
||||
// Legacy operations already recorded this same stage after Komodo deletion.
|
||||
deploymentStage(operation, REMOVING_SERVER);
|
||||
// Host directory. Fall back to derived path if host_dir wasn't set on
|
||||
// legacy rows.
|
||||
const hostDir = row?.host_dir ?? serverDir(name);
|
||||
try {
|
||||
await rm(hostDir, { recursive: true, force: true });
|
||||
} catch (err) {
|
||||
warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
// Remove server-owned beta and deployment rows before the registry row.
|
||||
// Test evidence cascades from runs; inventory and snapshots cascade from servers.
|
||||
try {
|
||||
const tx = db().transaction((n: string) => {
|
||||
db().prepare('DELETE FROM deployment_servers WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM deployment_files WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_servers WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_server_scripts WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_test_runs WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_test_coverage WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_file_changes WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(n);
|
||||
});
|
||||
tx(name);
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
kind: 'db',
|
||||
message: `failed to remove DB rows for ${name}: ${(err as Error).message}`,
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
const unfinished = db().prepare("SELECT id, action FROM server_deployments WHERE server = ? AND state = 'awaiting-health'").all(name) as { id: string; action: string }[];
|
||||
for (const { id, action } of unfinished) finishDeployment(id, { ok: false, message: action === 'create'
|
||||
? 'Server destroyed before creation health checks completed' : 'Server destroyed before operation verification completed' });
|
||||
return { ok: true, warnings };
|
||||
}
|
||||
interface CreationJob {
|
||||
deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
|
||||
}
|
||||
|
||||
async function provisionCreation(id: string): Promise<CreateServerResult> {
|
||||
const job = db().prepare('SELECT j.*, d.triggered_by FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE j.deployment_id = ?').get(id) as CreationJob;
|
||||
creating = true;
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(id);
|
||||
try {
|
||||
const mapEntry = zombieMapBySlug(gameOfServer(job.server), job.map_slug);
|
||||
const plutoKey = getServerKey(job.server);
|
||||
if (!mapEntry || !plutoKey) {
|
||||
const result = { ok: false as const, kind: 'bad-request' as const, message: 'Saved server key or selected map is no longer available' };
|
||||
finishDeployment(id, result);
|
||||
return result;
|
||||
}
|
||||
const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by },
|
||||
{ ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
|
||||
if (!result.ok) finishDeployment(id, result);
|
||||
else {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for Komodo health');
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
const server = getServer(job.server);
|
||||
if (server?.game_port === job.port) {
|
||||
// A lost response may have happened after Komodo started the server.
|
||||
// Keep its slot and observe readiness instead of starting another boot.
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message);
|
||||
deploymentStage(id, 'Verifying existing server after interrupted startup');
|
||||
} else finishDeployment(id, { ok: false, message: (err as Error).message });
|
||||
throw err;
|
||||
} finally { creating = false; }
|
||||
}
|
||||
|
||||
type CreationOutcomes = Map<string, CreateServerResult | Error>;
|
||||
let creationTick: Promise<CreationOutcomes> | null = null;
|
||||
function creationPass(): Promise<CreationOutcomes> {
|
||||
if (creationTick) return creationTick;
|
||||
creationTick = runCreations().finally(() => { creationTick = null; });
|
||||
return creationTick;
|
||||
}
|
||||
|
||||
export async function processCreations(): Promise<void> {
|
||||
await creationPass();
|
||||
}
|
||||
|
||||
/** One host-wide FIFO, shared by collections and individual launches. */
|
||||
export function creationQueue() {
|
||||
type Entry = { id: string; server: string; state: string; stage: string };
|
||||
const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments
|
||||
WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
|
||||
ORDER BY started_at, rowid`).all() as Entry[];
|
||||
return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
|
||||
}
|
||||
|
||||
function creationBlocker(id: string): string | null {
|
||||
const dependency = db().prepare(`SELECT d.server, d.state FROM server_creation_dependencies dep
|
||||
JOIN server_deployments d ON d.id = dep.removal_id
|
||||
WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { server: string; state: string } | undefined;
|
||||
if (dependency) return `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`;
|
||||
const occupied = db().prepare(`SELECT s.name FROM servers s JOIN server_creation_jobs j ON s.game_port = j.port
|
||||
WHERE j.deployment_id = ? LIMIT 1`).get(id) as { name: string } | undefined;
|
||||
return occupied ? `Waiting for port to be released by ${occupied.name}` : null;
|
||||
}
|
||||
|
||||
async function runCreations(): Promise<CreationOutcomes> {
|
||||
const outcomes: CreationOutcomes = new Map();
|
||||
if (creating || destroying.size > 0) return outcomes;
|
||||
// Restart recovery is observation-only: never repeat external creation or
|
||||
// deployment. A known interrupted startup must retain its readiness gate.
|
||||
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
|
||||
for (const { id } of interrupted) {
|
||||
if (resumeVerification(id)) continue;
|
||||
// A queued removal does not make the interrupted container disappear.
|
||||
// Keep its slot until removal finishes, without resuming normal work or
|
||||
// confusing it with a newer creation of the same server identity.
|
||||
const removing = db().prepare(`SELECT 1 FROM server_deployments d
|
||||
JOIN server_creation_jobs j ON j.deployment_id = d.id
|
||||
JOIN servers s ON s.name = j.server AND s.game_port = j.port
|
||||
WHERE d.id = ? AND EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server
|
||||
AND newer.rowid > d.rowid AND newer.action = 'destroy' AND newer.state IN ('waiting', 'running'))
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server
|
||||
AND newer.rowid > d.rowid AND newer.action = 'create')`).get(id);
|
||||
if (removing) {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for requested server removal after interrupted startup');
|
||||
}
|
||||
}
|
||||
|
||||
// Check only previously started servers. Newly launched servers keep their
|
||||
// slot until a later pass, even if a health probe would answer immediately.
|
||||
const checking = db().prepare(`SELECT j.* FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[];
|
||||
for (const job of checking) {
|
||||
const id = job.deployment_id;
|
||||
if (getServer(job.server)?.game_port !== job.port) {
|
||||
finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' });
|
||||
continue;
|
||||
}
|
||||
if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
|
||||
try {
|
||||
const komodo = await komodoInspectHealthy(job.server);
|
||||
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
|
||||
const rcon = await rconStatusResult(job.server);
|
||||
if (!rcon.ok) { deploymentStage(id, 'Waiting for RCON health: ' + rcon.reason); continue; }
|
||||
const actualMap = rcon.status.map;
|
||||
if (actualMap !== job.expected_map) {
|
||||
deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', RCON reports ' + (actualMap ?? 'no map'));
|
||||
continue;
|
||||
}
|
||||
deploymentStage(id, 'Verified Komodo healthy, RCON healthy, and map ' + actualMap);
|
||||
finishDeployment(id, { ok: true });
|
||||
} catch (err) {
|
||||
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot the budget once. Fast success/failure must not turn a single
|
||||
// worker pass into an unbounded stream of expensive container startups.
|
||||
const queue = creationQueue();
|
||||
const available = Math.max(0, config.createConcurrency - queue.active.length);
|
||||
const eligible = queue.queued.filter(({ id }) => {
|
||||
const blocker = creationBlocker(id);
|
||||
if (blocker) deploymentStage(id, blocker);
|
||||
return !blocker;
|
||||
});
|
||||
for (const { id } of eligible.slice(0, available)) {
|
||||
if (creating || destroying.size > 0) break;
|
||||
// API calls can cancel a selected job or resume an older startup while
|
||||
// a previous provision awaits Komodo. Recheck without expanding this pass.
|
||||
if (creationQueue().active.length >= config.createConcurrency) break;
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
|
||||
const blocker = creationBlocker(id);
|
||||
if (blocker) {
|
||||
deploymentStage(id, blocker);
|
||||
continue;
|
||||
}
|
||||
try { outcomes.set(id, await provisionCreation(id)); }
|
||||
catch (err) {
|
||||
const failure = err instanceof Error ? err : new Error(String(err));
|
||||
outcomes.set(id, failure);
|
||||
console.error('[creation] provisioning failed:', failure);
|
||||
}
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
export * from '../provisioning/instances.js';
|
||||
|
||||
@@ -1,133 +1 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { db } from '../db/index.js';
|
||||
import { recipeForDeployment, summarizeRecipe } from './launch-recipes.js';
|
||||
import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
|
||||
|
||||
export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
|
||||
const id = randomUUID();
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
||||
VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
|
||||
recordDeploymentEvent(id, action === 'create' ? 'Server creation requested'
|
||||
: action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested`);
|
||||
return id;
|
||||
}
|
||||
|
||||
export function deploymentStage(id: string, stage: string): void {
|
||||
const previous = db().prepare('SELECT stage FROM server_deployments WHERE id = ?').get(id) as { stage: string } | undefined;
|
||||
db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id);
|
||||
if (previous && previous.stage !== stage) recordDeploymentEvent(id, stage);
|
||||
}
|
||||
|
||||
export function recordDeploymentEvent(id: string, message: string): void {
|
||||
db().prepare('INSERT INTO server_deployment_events (deployment_id, created_at, message) VALUES (?, ?, ?)')
|
||||
.run(id, new Date().toISOString(), message);
|
||||
}
|
||||
|
||||
export function getDeployment(id: string) {
|
||||
const deployment = db().prepare(`SELECT d.*, j.expected_map, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
|
||||
FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id);
|
||||
if (!deployment) return null;
|
||||
const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id);
|
||||
const launch = recipeForDeployment(id);
|
||||
const collection = db().prepare(`SELECT c.id, c.name FROM (
|
||||
SELECT collection_id, deployment_id FROM server_collection_members
|
||||
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
|
||||
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ?`).get(id) ?? null;
|
||||
return { deployment, events, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
|
||||
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection };
|
||||
}
|
||||
|
||||
// This stage is written only after Komodo has confirmed deployment deletion.
|
||||
export const REMOVING_SERVER = 'Removing server files and registry';
|
||||
|
||||
export function canCancelRemoval(id: string): boolean {
|
||||
return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER);
|
||||
}
|
||||
|
||||
/** Competes atomically with the worker's claim, before any external removal. */
|
||||
export function cancelRemoval(id: string): boolean {
|
||||
return db().transaction(() => {
|
||||
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
|
||||
WHERE id = ? AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(new Date().toISOString(), id, REMOVING_SERVER);
|
||||
if (!result.changes) return false;
|
||||
recordDeploymentEvent(id, 'Queued removal cancelled. The server and its files were kept.');
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
|
||||
export function claimRemoval(id: string): boolean {
|
||||
return db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND action = 'destroy' AND state = 'waiting'").run(id).changes > 0;
|
||||
}
|
||||
|
||||
export function canResumeCleanup(id: string): boolean {
|
||||
return !!db().prepare(`SELECT d.id FROM server_deployments d
|
||||
WHERE d.id = ? AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
||||
.get(id, REMOVING_SERVER);
|
||||
}
|
||||
|
||||
export function resumeCleanup(id: string): boolean {
|
||||
return db().transaction(() => {
|
||||
if (!canResumeCleanup(id)) return false;
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
recordDeploymentEvent(id, 'Resumed local cleanup after confirmed Komodo deletion');
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
|
||||
export function canResumeVerification(id: string): boolean {
|
||||
return !!db().prepare(`SELECT d.id FROM server_deployments d
|
||||
JOIN server_creation_jobs j ON j.deployment_id = d.id
|
||||
JOIN servers s ON s.name = j.server AND s.game_port = j.port
|
||||
WHERE d.id = ? AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
|
||||
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL))) AND d.state IN ('failed', 'interrupted')
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
||||
.get(id);
|
||||
}
|
||||
|
||||
export function canResumeRebuild(id: string): boolean {
|
||||
return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
|
||||
WHERE d.id = ? AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
||||
.get(id, REMOVING_SERVER);
|
||||
}
|
||||
|
||||
export function resumeRebuild(id: string): boolean {
|
||||
if (!canResumeRebuild(id)) return false;
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
recordDeploymentEvent(id, 'Resumed rebuild cleanup after confirmed Komodo deletion');
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Resume observation only. Never repeat a container creation or restart. */
|
||||
export function resumeVerification(id: string): boolean {
|
||||
return db().transaction(() => {
|
||||
if (!canResumeVerification(id)) return false;
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
recordDeploymentEvent(id, 'Resumed verification of the existing server');
|
||||
deploymentStage(id, 'Waiting for Komodo health');
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
|
||||
export function finishDeployment(id: string, result: { ok: boolean; message?: string; warnings?: string[] }): void {
|
||||
const state = !result.ok ? 'failed' : result.warnings?.length ? 'warning' : 'done';
|
||||
db().prepare(`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`)
|
||||
.run(state, new Date().toISOString(), result.message ?? null, JSON.stringify(result.warnings ?? []), id);
|
||||
for (const warning of result.warnings ?? []) recordDeploymentEvent(id, warning);
|
||||
recordDeploymentEvent(id, result.ok ? (state === 'warning' ? 'Completed with warnings' : 'Completed') : `Failed: ${result.message ?? 'Unknown error'}`);
|
||||
}
|
||||
|
||||
export function listDeployments() {
|
||||
return db().prepare(`SELECT d.*, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists,
|
||||
(d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval
|
||||
FROM server_deployments d ORDER BY started_at DESC, rowid DESC`).all(REMOVING_SERVER);
|
||||
}
|
||||
|
||||
export function interruptDeployments(): void {
|
||||
const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running'").all() as { id: string }[];
|
||||
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?,
|
||||
error = 'The application restarted before this operation finished. Check the server before retrying.'
|
||||
WHERE state = 'running'`).run(new Date().toISOString());
|
||||
for (const { id } of running) recordDeploymentEvent(id, 'Interrupted by application restart. Check the server before retrying.');
|
||||
}
|
||||
export * from '../provisioning/history.js';
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { destroyServer } from './create.js';
|
||||
import { destroyServer } from '../provisioning/index.js';
|
||||
|
||||
export function housePartyTargets(): string[] {
|
||||
return (db().prepare(`
|
||||
|
||||
@@ -1,117 +1,2 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { getServer } from './servers.js';
|
||||
import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER } from './deployments.js';
|
||||
import { queueRebuildServer, rebuildServer } from './create.js';
|
||||
import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
|
||||
import { rconGetPlayerCount } from '../integrations/rcon.js';
|
||||
import { refreshRestartRollout, restartInRollout, rolloutWaitReason } from '../pipeline/restart-rollout.js';
|
||||
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
|
||||
|
||||
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
|
||||
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
|
||||
|
||||
function blocker(server: string, id = ''): string | null {
|
||||
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ?
|
||||
AND state IN ('queued','waiting','running','awaiting-health')`).get(server, id)) return 'Waiting for another provisioning operation on this server';
|
||||
if (db().prepare(`SELECT 1 FROM deployment_servers WHERE server = ? AND restart_state IN ('pending','failed','held')
|
||||
UNION ALL SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
|
||||
WHERE bs.server = ? AND b.state IN ('running','ready-promote')
|
||||
UNION ALL SELECT 1 FROM maintenance_jobs WHERE server = ? AND state IN ('queued','restarting') LIMIT 1`).get(server, server, server)) {
|
||||
return 'Waiting for campaign, beta, or maintenance work on this server';
|
||||
}
|
||||
return rolloutWaitReason(server, id);
|
||||
}
|
||||
|
||||
export function requestLifecycle(server: string, action: RuntimeAction) {
|
||||
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
|
||||
if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' };
|
||||
return db().transaction(() => {
|
||||
const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
|
||||
AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
|
||||
if (existing) return existing.action === action
|
||||
? { ok: true as const, deploymentId: existing.id }
|
||||
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
|
||||
const row = getServer(server);
|
||||
if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' };
|
||||
if (action === 'rebuild') {
|
||||
const result = queueRebuildServer(server);
|
||||
return result.ok ? { ok: true as const, deploymentId: result.deploymentId }
|
||||
: { ok: false as const, status: 400, error: result.message };
|
||||
}
|
||||
const id = beginDeployment(server, action);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty');
|
||||
return { ok: true as const, deploymentId: id };
|
||||
})();
|
||||
}
|
||||
|
||||
interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
|
||||
let ticking: Promise<void> | null = null;
|
||||
export function processLifecycle(): Promise<void> {
|
||||
if (ticking) return ticking;
|
||||
ticking = run().finally(() => { ticking = null; });
|
||||
return ticking;
|
||||
}
|
||||
|
||||
async function run(): Promise<void> {
|
||||
await refreshRestartRollout();
|
||||
const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments
|
||||
WHERE action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
|
||||
ORDER BY started_at, rowid`).all() as Operation[];
|
||||
for (const operation of operations) {
|
||||
try { await processOperation(operation); }
|
||||
catch (err) {
|
||||
// Observation failures are retried; remote mutations are never replayed.
|
||||
deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function processOperation(operation: Operation): Promise<void> {
|
||||
const { id, server, action } = operation;
|
||||
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
|
||||
if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
|
||||
if (!komodoConfigured() || !rconConfigured()) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; }
|
||||
if (operation.state === 'awaiting-health') {
|
||||
const health = await komodoInspectHealthy(server);
|
||||
if (isStopped(health.state)) {
|
||||
db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
|
||||
deploymentStage(id, 'Verified container stopped');
|
||||
finishDeployment(id, { ok: true });
|
||||
} else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
|
||||
return;
|
||||
}
|
||||
const reason = blocker(server, id);
|
||||
if (reason) { deploymentStage(id, reason); return; }
|
||||
const health = await komodoInspectHealthy(server);
|
||||
if (action !== 'start' && !isStopped(health.state)) {
|
||||
const players = await rconGetPlayerCount(server);
|
||||
if (players !== 0) {
|
||||
deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Requests can arrive during the probes. Recheck before claiming the action.
|
||||
if (blocker(server, id)) return;
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'waiting'").get(id)) return;
|
||||
if (action === 'rebuild') { await rebuildServer(id); return; }
|
||||
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
|
||||
if (!claim.changes) return;
|
||||
if (action === 'start' || action === 'restart') {
|
||||
const result = await restartInRollout(server, { operationId: id, action });
|
||||
if (result.kind === 'queued') {
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
deploymentStage(id, result.message);
|
||||
}
|
||||
return;
|
||||
}
|
||||
deploymentStage(id, 'Stopping container');
|
||||
// Persist intent first; a lost reply must not issue another stop later.
|
||||
try {
|
||||
const result = await komodoRuntimeAction(server, 'stop');
|
||||
if (!result.ok) { finishDeployment(id, { ok: false, message: result.message }); return; }
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for container to stop');
|
||||
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
|
||||
}
|
||||
export { requestLifecycle } from '../provisioning/requests.js';
|
||||
export { processLifecycle } from '../provisioning/worker.js';
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { config, komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { config } from '../config.js';
|
||||
import { getServer, listServers } from './servers.js';
|
||||
import { rconGetPlayerCount, rconPlutoniumVersion } from '../integrations/rcon.js';
|
||||
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from '../pipeline/restart-rollout.js';
|
||||
import { rconPlutoniumVersion } from '../integrations/rcon.js';
|
||||
import { requestRestart, cancelProvisioning, restartRecoveryPending } from '../provisioning/index.js';
|
||||
import { nextMaintenanceAt, parsePlutoniumManifest, PLUTONIUM_MANIFEST_URL } from '../../plutonium.js';
|
||||
import type { MaintenanceJob, MaintenanceSchedule, PlutoniumRelease, PlutoniumStatus, PlutoniumVersion } from '../../plutonium.js';
|
||||
|
||||
@@ -39,7 +39,7 @@ export function plutoniumStatus(): PlutoniumStatus {
|
||||
}
|
||||
|
||||
async function probeVersion(server: string): Promise<number | null> {
|
||||
const recovering = db().prepare('SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL').get(server);
|
||||
const recovering = restartRecoveryPending(server);
|
||||
const result = recovering ? { revision: null, error: 'Waiting for restart recovery.' }
|
||||
: await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' }));
|
||||
// Clear the old revision on failure: it is no longer evidence of what is running.
|
||||
@@ -84,21 +84,22 @@ export function queueMaintenance(servers: string[]): number {
|
||||
for (const server of new Set(servers)) {
|
||||
added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at)
|
||||
VALUES (?, ?, ?)`).run(server, target, nowIso()).changes;
|
||||
const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND state IN ('queued','restarting')").get(server) as { id: number };
|
||||
requestRestart(server, { kind: 'maintenance', id: String(job.id) });
|
||||
}
|
||||
return added;
|
||||
})();
|
||||
}
|
||||
export function cancelMaintenance(id: number): boolean {
|
||||
return db().prepare(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator'
|
||||
WHERE id = ? AND state = 'queued'`).run(nowIso(), id).changes > 0;
|
||||
return db().transaction(() => {
|
||||
const job = db().prepare("SELECT provisioning_id FROM maintenance_jobs WHERE id = ? AND state = 'queued'").get(id) as { provisioning_id: string | null } | undefined;
|
||||
if (!job) return false;
|
||||
if (job.provisioning_id && !cancelProvisioning(job.provisioning_id)) return false;
|
||||
return db().prepare(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator'
|
||||
WHERE id = ? AND state = 'queued'`).run(nowIso(), id).changes > 0;
|
||||
})();
|
||||
}
|
||||
|
||||
function conflictingWork(server: string): boolean {
|
||||
return !!db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND state IN ('queued','running','awaiting-health')
|
||||
UNION ALL SELECT 1 FROM deployment_servers WHERE server = ? AND restart_state IN ('pending','failed')
|
||||
UNION ALL SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
|
||||
WHERE bs.server = ? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server, server, server);
|
||||
}
|
||||
function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) {
|
||||
db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ?')
|
||||
.run(state, detail, nowIso(), job.id);
|
||||
@@ -125,7 +126,6 @@ async function runTick(): Promise<void> {
|
||||
writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) });
|
||||
}
|
||||
})();
|
||||
await refreshRestartRollout();
|
||||
const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[];
|
||||
for (const job of jobs) {
|
||||
try { await processJob(job); }
|
||||
@@ -137,37 +137,20 @@ async function runTick(): Promise<void> {
|
||||
async function processJob(job: MaintenanceJob): Promise<void> {
|
||||
const server = getServer(job.server);
|
||||
if (!server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
|
||||
if (job.state === 'restarting') {
|
||||
const slot = db().prepare('SELECT requested_at FROM restart_rollout WHERE server = ?').get(job.server) as { requested_at: string } | undefined;
|
||||
if (!slot || !job.restarted_at || slot.requested_at < job.restarted_at) {
|
||||
return finish(job, 'failed', 'Restart submission was interrupted. Inspect the server before queuing another restart.');
|
||||
}
|
||||
const health = rolloutHealth(job.server);
|
||||
if (!health.ok) return hold(job, health.reason);
|
||||
const revision = await probeVersion(job.server);
|
||||
if (job.target_revision && (revision === null || revision < job.target_revision)) {
|
||||
return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`);
|
||||
}
|
||||
return finish(job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`);
|
||||
const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) });
|
||||
if (['failed','interrupted','cancelled'].includes(operation.state)) {
|
||||
return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage);
|
||||
}
|
||||
if (!komodoConfigured() || !rconConfigured()) return hold(job, 'Configure Komodo and RCON before restarting.');
|
||||
if (conflictingWork(job.server)) return hold(job, 'Waiting for deployment or beta work on this server.');
|
||||
const reason = rolloutWaitReason(job.server);
|
||||
if (reason) return hold(job, reason);
|
||||
const players = await rconGetPlayerCount(job.server);
|
||||
if (players === null || players > 0) return hold(job, players === null ? 'Waiting for a fresh RCON player count.' : `Waiting for ${players} player(s) to leave.`);
|
||||
// Recheck after async I/O; cancellation and lifecycle operations may have arrived.
|
||||
const current = getServer(job.server);
|
||||
if (!current || current.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
|
||||
if (conflictingWork(job.server)) return hold(job, 'Waiting for deployment or beta work on this server.');
|
||||
const claimed = db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ?, detail = 'Restart requested'
|
||||
WHERE id = ? AND state = 'queued'`).run(nowIso(), job.id);
|
||||
if (!claimed.changes) return;
|
||||
db().prepare('UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?')
|
||||
.run('Waiting for restart recovery.', job.server);
|
||||
// Persist intent before remote I/O. An ambiguous timeout must never cause an automatic second restart.
|
||||
const result = await restartInRollout(job.server);
|
||||
if (result.kind === 'queued') {
|
||||
db().prepare(`UPDATE maintenance_jobs SET state = 'queued', restarted_at = NULL, detail = ? WHERE id = ?`).run(result.message, job.id);
|
||||
} else if (result.kind === 'failed') finish(job, 'failed', result.message);
|
||||
if (operation.state !== 'done') {
|
||||
if (operation.submitted_at) {
|
||||
db().prepare("UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ? WHERE id = ?").run(operation.submitted_at, job.id);
|
||||
db().prepare('UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?').run('Waiting for restart recovery.', job.server);
|
||||
}
|
||||
return hold(job, operation.stage);
|
||||
}
|
||||
const revision = await probeVersion(job.server);
|
||||
if (job.target_revision && (revision === null || revision < job.target_revision)) {
|
||||
return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`);
|
||||
}
|
||||
return finish(job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`);
|
||||
}
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { rconGetPlayerCount } from '../integrations/rcon.js';
|
||||
import { betaOperationInFlight, deployBetaServers, getBeta, listBetaServers, rollbackBeta } from '../fleet/beta.js';
|
||||
import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js';
|
||||
import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js';
|
||||
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
|
||||
import { emit, type BetaStage } from './bus.js';
|
||||
import { restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
|
||||
import { requestRestart, releaseRestartRequest } from '../provisioning/index.js';
|
||||
|
||||
function stage(id: number, server: string, next: BetaStage, error: string | null = null): void {
|
||||
db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ?')
|
||||
@@ -80,11 +78,6 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
|
||||
setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence');
|
||||
return;
|
||||
}
|
||||
if (!komodoConfigured() || !rconConfigured()) {
|
||||
betaState(id, false);
|
||||
setOutcome(id, 'active', 'Waiting for Komodo and RCON configuration');
|
||||
return;
|
||||
}
|
||||
for (const server of servers) {
|
||||
const freshPlan = storedPlan(id)!;
|
||||
if (!['active', 'passed'].includes(freshPlan.outcome)) return;
|
||||
@@ -102,35 +95,33 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
|
||||
const broken = evaluateRequirements(plan, readMetrics(currentRun(id, server.server)!), waitingRun.phase).find(p => p.failed);
|
||||
if (broken) { await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; }
|
||||
}
|
||||
if (rolloutWaitReason(server.server)) continue;
|
||||
if (await rconGetPlayerCount(server.server) !== 0) continue;
|
||||
const oldRun = currentRun(id, server.server);
|
||||
const phase = oldRun ? oldRun.phase + 1 : 0;
|
||||
// Fail closed after interrupted restart requests; never silently reuse earlier evidence.
|
||||
stage(id, server.server, 'restarting');
|
||||
const run = await prepareRun(id, server.server, phase);
|
||||
if (oldRun) {
|
||||
const previous = readMetrics(oldRun);
|
||||
const metrics = readMetrics(run);
|
||||
metrics.outages = previous.outages;
|
||||
saveMetrics(run, metrics);
|
||||
// Prepare evidence once before handing off the restart. Empty-server
|
||||
// checks and remote execution belong entirely to Provisioning.
|
||||
let run = currentRun(id, server.server);
|
||||
if (!run || run.observing_at) {
|
||||
const oldRun = run;
|
||||
run = await prepareRun(id, server.server, oldRun ? oldRun.phase + 1 : 0);
|
||||
if (oldRun) {
|
||||
const metrics = readMetrics(run);
|
||||
metrics.outages = readMetrics(oldRun).outages;
|
||||
saveMetrics(run, metrics);
|
||||
}
|
||||
}
|
||||
const restart = await restartInRollout(server.server);
|
||||
if (restart.kind === 'queued') {
|
||||
db().prepare('DELETE FROM beta_test_runs WHERE id = ?').run(run.id);
|
||||
if (oldRun) db().prepare('UPDATE beta_test_runs SET retired_at = NULL WHERE id = ?').run(oldRun.id);
|
||||
stage(id, server.server, 'awaiting-empty', restart.message); continue;
|
||||
}
|
||||
if (restart.kind === 'failed') { stage(id, server.server, 'failed', restart.message); await fail(id, plan, restart.message); return; }
|
||||
db().prepare('UPDATE beta_servers SET restarted_at = ?, healthy_at = NULL WHERE beta_id = ? AND server = ?')
|
||||
.run(new Date().toISOString(), id, server.server);
|
||||
stage(id, server.server, 'awaiting-health');
|
||||
continue;
|
||||
}
|
||||
if (server.stage === 'awaiting-health') {
|
||||
const health = rolloutHealth(server.server);
|
||||
if (!health.ok) continue;
|
||||
const now = new Date().toISOString();
|
||||
if (server.stage === 'awaiting-empty' || server.stage === 'awaiting-health') {
|
||||
const operation = requestRestart(server.server, { kind: 'beta', id: String(id) });
|
||||
if (['failed','interrupted','cancelled'].includes(operation.state)) {
|
||||
const reason = operation.error ?? operation.stage;
|
||||
stage(id, server.server, 'failed', reason); await fail(id, plan, reason); return;
|
||||
}
|
||||
if (!operation.submitted_at || operation.state === 'running') {
|
||||
stage(id, server.server, 'awaiting-empty', operation.stage);
|
||||
continue;
|
||||
}
|
||||
db().prepare('UPDATE beta_servers SET restarted_at = ? WHERE beta_id = ? AND server = ?')
|
||||
.run(operation.submitted_at, id, server.server);
|
||||
if (operation.state !== 'done') { stage(id, server.server, 'awaiting-health', operation.stage); continue; }
|
||||
const now = operation.finished_at ?? new Date().toISOString();
|
||||
db().prepare('UPDATE beta_servers SET healthy_at = ? WHERE beta_id = ? AND server = ?').run(now, id, server.server);
|
||||
db().prepare('UPDATE beta_test_runs SET observing_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(now, id, server.server);
|
||||
stage(id, server.server, 'observing');
|
||||
@@ -145,6 +136,8 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
|
||||
if (broken) { stage(id, server.server, 'failed', broken.detail); await fail(id, plan, `${server.server}: ${broken.label}: ${broken.detail}`); return; }
|
||||
const recoveryTarget = Math.max(0, ...plan.requirements.filter(r => r.kind === 'recovery').map(r => r.target));
|
||||
if (run.phase < recoveryTarget && progress.filter(p => !plan.requirements.some(r => r.id === p.id && r.kind === 'recovery')).every(p => p.passed)) {
|
||||
releaseRestartRequest(server.server, { kind: 'beta', id: String(id) });
|
||||
db().prepare('UPDATE beta_servers SET restarted_at = NULL, healthy_at = NULL WHERE beta_id = ? AND server = ?').run(id, server.server);
|
||||
stage(id, server.server, 'awaiting-empty');
|
||||
activity(id, `${server.server}: recovery cycle ${run.phase + 1}/${recoveryTarget} queued; gameplay requirements will be repeated`);
|
||||
} else stage(id, server.server, progress.every(p => p.passed) ? 'ready' : 'observing');
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { emit } from './bus.js';
|
||||
import type { BetaStage } from './bus.js';
|
||||
import { komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { rconGetPlayerCount } from '../integrations/rcon.js';
|
||||
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
|
||||
import { requestRestart } from '../provisioning/index.js';
|
||||
import { plannedBetaTick } from './beta-plan-runner.js';
|
||||
|
||||
interface BetaServerActiveRow {
|
||||
@@ -20,10 +18,6 @@ function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function integrationsOn(): boolean {
|
||||
return komodoConfigured() && rconConfigured();
|
||||
}
|
||||
|
||||
const setStage = () =>
|
||||
db().prepare(
|
||||
`UPDATE beta_servers
|
||||
@@ -94,9 +88,7 @@ export function betaTick(): Promise<{ processed: number }> {
|
||||
}
|
||||
|
||||
async function runBetaTick(): Promise<{ processed: number }> {
|
||||
if (integrationsOn()) await refreshRestartRollout();
|
||||
await plannedBetaTick();
|
||||
if (!integrationsOn()) return { processed: 0 };
|
||||
|
||||
const rows = db()
|
||||
.prepare(
|
||||
@@ -131,49 +123,21 @@ async function runBetaTick(): Promise<{ processed: number }> {
|
||||
async function advance(row: BetaServerActiveRow): Promise<void> {
|
||||
switch (row.stage) {
|
||||
case 'awaiting-empty':
|
||||
case 'restarting': {
|
||||
const waitReason = rolloutWaitReason(row.server);
|
||||
if (waitReason) {
|
||||
transition(row.beta_id, row.server, 'awaiting-empty', waitReason);
|
||||
return;
|
||||
}
|
||||
const players = await rconGetPlayerCount(row.server);
|
||||
if (players === null) return; // hold — RCON unreachable, try next tick
|
||||
if (players > 0) return; // hold — still populated
|
||||
transition(row.beta_id, row.server, 'restarting');
|
||||
const r = await restartInRollout(row.server);
|
||||
if (r.kind === 'queued') {
|
||||
transition(row.beta_id, row.server, 'awaiting-empty', r.message);
|
||||
return;
|
||||
}
|
||||
if (r.kind === 'failed') {
|
||||
transition(row.beta_id, row.server, 'failed', r.message);
|
||||
return;
|
||||
}
|
||||
const next = row.wait_for_health ? 'awaiting-health' : row.wait_for_match ? 'awaiting-match' : 'ready';
|
||||
transition(row.beta_id, row.server, next);
|
||||
return;
|
||||
}
|
||||
case 'restarting':
|
||||
case 'awaiting-health': {
|
||||
const h = rolloutHealth(row.server);
|
||||
if (!h.ok) {
|
||||
// Record the failure reason (including any snippet) so the
|
||||
// beta detail UI can surface why we're stuck.
|
||||
setStage().run(
|
||||
'awaiting-health',
|
||||
h.reason,
|
||||
nowIso(),
|
||||
'awaiting-health',
|
||||
'awaiting-health',
|
||||
nowIso(),
|
||||
'awaiting-health',
|
||||
'awaiting-health',
|
||||
nowIso(),
|
||||
'awaiting-health',
|
||||
nowIso(),
|
||||
row.beta_id,
|
||||
row.server
|
||||
);
|
||||
const operation = requestRestart(row.server, { kind: 'beta', id: String(row.beta_id) });
|
||||
if (['failed','interrupted','cancelled'].includes(operation.state)) {
|
||||
transition(row.beta_id, row.server, 'failed', operation.error ?? operation.stage);
|
||||
return;
|
||||
}
|
||||
if (!operation.submitted_at || operation.state === 'running') {
|
||||
transition(row.beta_id, row.server, 'awaiting-empty', operation.stage);
|
||||
return;
|
||||
}
|
||||
db().prepare('UPDATE beta_servers SET restarted_at = ? WHERE beta_id = ? AND server = ?')
|
||||
.run(operation.submitted_at, row.beta_id, row.server);
|
||||
if (row.wait_for_health && operation.state !== 'done') {
|
||||
transition(row.beta_id, row.server, 'awaiting-health', operation.stage);
|
||||
return;
|
||||
}
|
||||
const next = row.wait_for_match ? 'awaiting-match' : 'ready';
|
||||
|
||||
@@ -1,157 +1 @@
|
||||
import { config } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
|
||||
import { rconStatusResult } from '../integrations/rcon.js';
|
||||
import { beginDeployment, deploymentStage, finishDeployment } from '../fleet/deployments.js';
|
||||
|
||||
// Komodo queues restart work. Don't mistake the still-running old container
|
||||
// for a completed restart when an API kick immediately follows the request.
|
||||
const HEALTH_POLL_MS = 15_000;
|
||||
const requestsInFlight = new Set<string>();
|
||||
|
||||
interface RestartSlot {
|
||||
server: string;
|
||||
requested_at: string;
|
||||
checked_at: string | null;
|
||||
healthy_at: string | null;
|
||||
players: number | null;
|
||||
error: string | null;
|
||||
operation_id: string | null;
|
||||
}
|
||||
|
||||
function slotFor(server: string): RestartSlot | undefined {
|
||||
return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
|
||||
}
|
||||
|
||||
/** The queued restart may still expose the previous healthy container. */
|
||||
export function restartSettling(server: string): boolean {
|
||||
const slot = slotFor(server);
|
||||
return requestsInFlight.has(server) || !!(slot && !slot.healthy_at
|
||||
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
|
||||
}
|
||||
|
||||
export function rolloutWaitReason(server: string, operationId = ''): string | null {
|
||||
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ?
|
||||
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
|
||||
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) {
|
||||
return 'Waiting for the current provisioning operation';
|
||||
}
|
||||
const slot = slotFor(server);
|
||||
if (slot?.healthy_at === null) {
|
||||
return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
|
||||
}
|
||||
const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
|
||||
return n >= config.restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null;
|
||||
}
|
||||
|
||||
export type RolloutRestartResult =
|
||||
| { kind: 'started' }
|
||||
| { kind: 'queued'; message: string }
|
||||
| { kind: 'failed'; message: string };
|
||||
|
||||
/** Reserve capacity before any async I/O; callers still check for players first. */
|
||||
export async function restartInRollout(server: string, options: { operationId?: string; action?: 'start' | 'restart'; triggeredBy?: string } = {}): Promise<RolloutRestartResult> {
|
||||
let operationId = options.operationId;
|
||||
const waitReason = db().transaction(() => {
|
||||
const reason = rolloutWaitReason(server, operationId);
|
||||
if (reason) return reason;
|
||||
operationId ??= beginDeployment(server, options.action ?? 'restart', options.triggeredBy ?? 'automation');
|
||||
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
|
||||
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
|
||||
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
|
||||
.run(server, new Date().toISOString(), operationId);
|
||||
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container');
|
||||
return null;
|
||||
}).immediate();
|
||||
if (waitReason) return { kind: 'queued', message: waitReason };
|
||||
|
||||
requestsInFlight.add(server);
|
||||
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
|
||||
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
|
||||
try {
|
||||
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
|
||||
if (result.ok) {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(operationId);
|
||||
deploymentStage(operationId!, 'Waiting for container and RCON health');
|
||||
return { kind: 'started' };
|
||||
}
|
||||
// These failures cannot have queued a remote restart. Other failures
|
||||
// (especially a timeout) may have: retain capacity until recovery.
|
||||
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
|
||||
db().prepare('DELETE FROM restart_rollout WHERE server = ?').run(server);
|
||||
} else {
|
||||
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server);
|
||||
}
|
||||
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
|
||||
return { kind: 'failed', message: `komodo: ${result.message}` };
|
||||
} catch (err) {
|
||||
const message = `komodo: ${(err as Error).message}`;
|
||||
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server);
|
||||
finishDeployment(operationId!, { ok: false, message });
|
||||
return { kind: 'failed', message };
|
||||
} finally {
|
||||
// Start the grace period at completion, including slow/failed requests.
|
||||
db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ?')
|
||||
.run(new Date().toISOString(), server);
|
||||
requestsInFlight.delete(server);
|
||||
}
|
||||
}
|
||||
|
||||
let refreshInFlight: Promise<void> | null = null;
|
||||
|
||||
/** Poll all occupied slots, even if their originating beta was already closed. */
|
||||
export function refreshRestartRollout(): Promise<void> {
|
||||
if (refreshInFlight) return refreshInFlight;
|
||||
refreshInFlight = refresh().finally(() => { refreshInFlight = null; });
|
||||
return refreshInFlight;
|
||||
}
|
||||
|
||||
async function refresh(): Promise<void> {
|
||||
// Adopt work started before this feature was installed. INSERT OR IGNORE
|
||||
// preserves completed slots and never resets a newer restart's progress.
|
||||
db().prepare(`INSERT OR IGNORE INTO restart_rollout (server, requested_at)
|
||||
SELECT server, MAX(restarted_at) FROM deployment_servers
|
||||
WHERE restart_state IN ('pending','failed') AND restarted_at IS NOT NULL GROUP BY server`).run();
|
||||
db().prepare(`INSERT OR IGNORE INTO restart_rollout (server, requested_at)
|
||||
SELECT bs.server, MAX(COALESCE(bs.restarted_at, bs.updated_at))
|
||||
FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
|
||||
WHERE b.state = 'running' AND bs.stage IN ('restarting','awaiting-health') GROUP BY bs.server`).run();
|
||||
|
||||
const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server')
|
||||
.all() as RestartSlot[];
|
||||
for (const slot of slots) {
|
||||
if (requestsInFlight.has(slot.server)) continue;
|
||||
if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
|
||||
let error: string | null = null;
|
||||
let players: number | null = null;
|
||||
try {
|
||||
const health = await komodoInspectHealthy(slot.server);
|
||||
if (!health.ok) {
|
||||
error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
|
||||
} else {
|
||||
const rcon = await rconStatusResult(slot.server);
|
||||
if (!rcon.ok) error = `rcon not answering: ${rcon.reason}`;
|
||||
else players = rcon.status.players;
|
||||
}
|
||||
} catch (err) {
|
||||
error = `health check: ${(err as Error).message}`;
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
|
||||
WHERE server = ? AND requested_at = ? AND healthy_at IS NULL`)
|
||||
.run(now, error ? null : now, players, error, slot.server, slot.requested_at);
|
||||
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'awaiting-health'").get(slot.operation_id)) {
|
||||
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
|
||||
else {
|
||||
deploymentStage(slot.operation_id, 'Verified container and RCON health');
|
||||
finishDeployment(slot.operation_id, { ok: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } {
|
||||
const slot = slotFor(server);
|
||||
if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 };
|
||||
return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' };
|
||||
}
|
||||
export { restartSettling, rolloutWaitReason, refreshRestartRollout, rolloutHealth } from '../provisioning/restarts.js';
|
||||
|
||||
@@ -1,202 +1,49 @@
|
||||
import { config, komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { config } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { emit } from './bus.js';
|
||||
import { rconGetPlayerCount } from '../integrations/rcon.js';
|
||||
import { setServerField } from '../fleet/servers.js';
|
||||
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from './restart-rollout.js';
|
||||
import { requestRestart } from '../provisioning/index.js';
|
||||
|
||||
interface PendingRow {
|
||||
deployment_id: string;
|
||||
server: string;
|
||||
attempts: number;
|
||||
restarted_at: string | null;
|
||||
files_changed: number;
|
||||
deployment_id: string; server: string; restart_state: string;
|
||||
restarted_at: string | null; files_changed: number;
|
||||
}
|
||||
|
||||
type SweepResult = { checked: number; restarted: number; healthy: number; held: number; failed: number };
|
||||
let sweepInFlight: Promise<SweepResult> | null = null;
|
||||
const nowIso = () => new Date().toISOString();
|
||||
|
||||
function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function integrationsOn(): boolean {
|
||||
return komodoConfigured() && rconConfigured();
|
||||
}
|
||||
|
||||
/**
|
||||
* Two-phase sweep of every pending/failed deployment_server row.
|
||||
*
|
||||
* Phase 1 (restarted_at IS NULL): the "wait for empty then restart" phase.
|
||||
* - Poll RCON for player count. Busy or unreachable → stay pending.
|
||||
* - Empty → call Komodo restart, set restarted_at, stay pending so the
|
||||
* row picks up phase 2 on the next tick.
|
||||
*
|
||||
* Phase 2 (restarted_at IS NOT NULL): the "wait for healthy" phase.
|
||||
* - Ask Komodo whether the container is running AND probe RCON.
|
||||
* - Both green → done. Either red → stay pending; keep polling.
|
||||
*
|
||||
* Same-server dedup is per-tick per-phase so multiple concurrent deployments
|
||||
* targeting the same server only cost one RCON+Komodo round trip per tick.
|
||||
*/
|
||||
/** Submit activation requests and mirror their results. Provisioning owns all execution. */
|
||||
export function sweep(): Promise<SweepResult> {
|
||||
if (sweepInFlight) return sweepInFlight;
|
||||
sweepInFlight = runSweep().finally(() => { sweepInFlight = null; });
|
||||
return sweepInFlight;
|
||||
}
|
||||
|
||||
async function runSweep(): Promise<SweepResult> {
|
||||
if (!integrationsOn()) {
|
||||
// Freeze pending rows as manual — they won't drain without integrations.
|
||||
db()
|
||||
.prepare(
|
||||
`UPDATE deployment_servers
|
||||
SET restart_state = 'manual', checked_at = ?, error = 'integrations not configured'
|
||||
WHERE restart_state IN ('pending','failed')`
|
||||
)
|
||||
.run(nowIso());
|
||||
emit({ type: 'sweep-tick', frozen: false, reason: 'integrations not configured' });
|
||||
return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
|
||||
}
|
||||
|
||||
// Recovery frees capacity before admitting any more servers. This runs
|
||||
// even without pending deployment rows (betas share the same slots).
|
||||
await refreshRestartRollout();
|
||||
const pending = db()
|
||||
.prepare(
|
||||
`SELECT ds.deployment_id, ds.server, ds.attempts, ds.restarted_at, ds.files_changed
|
||||
FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
|
||||
WHERE ds.restart_state IN ('pending','failed')
|
||||
ORDER BY d.started_at, d.id, ds.server`
|
||||
)
|
||||
.all() as PendingRow[];
|
||||
if (pending.length === 0) {
|
||||
emit({ type: 'sweep-tick', frozen: false });
|
||||
return { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
|
||||
}
|
||||
|
||||
type Outcome = { state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean };
|
||||
// Split dedup caches by phase — the two phases talk to different endpoints.
|
||||
const preRestart = new Map<string, Outcome>();
|
||||
const postRestart = new Map<string, Outcome>();
|
||||
const recoveringServers = new Set(pending.filter((row) => row.restarted_at).map((row) => row.server));
|
||||
|
||||
let checked = 0;
|
||||
let restarted = 0;
|
||||
let healthy = 0;
|
||||
let held = 0;
|
||||
let failed = 0;
|
||||
|
||||
const applyOutcome = db().prepare(
|
||||
`UPDATE deployment_servers
|
||||
SET restart_state = ?, checked_at = ?, players = ?, error = ?, attempts = attempts + 1,
|
||||
restarted_at = CASE WHEN ? = 1 THEN ? ELSE restarted_at END
|
||||
WHERE deployment_id = ? AND server = ?`
|
||||
);
|
||||
|
||||
const pending = db().prepare(`SELECT ds.* FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
|
||||
WHERE ds.restart_state IN ('pending','failed') ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[];
|
||||
const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
|
||||
for (const row of pending) {
|
||||
// A failed file write isn't a restart request. Preserve the original
|
||||
// failure until the operator repairs/retries that deployment.
|
||||
if (row.files_changed === 0) continue;
|
||||
const cache = row.restarted_at ? postRestart : preRestart;
|
||||
let outcome = cache.get(row.server);
|
||||
if (!outcome) {
|
||||
checked++;
|
||||
outcome = row.restarted_at
|
||||
? runHealthPhase(row.server)
|
||||
: recoveringServers.has(row.server)
|
||||
? { state: 'pending', players: null, error: 'Queued behind an earlier restart of this server', restartedNow: false }
|
||||
: await runRestartPhase(row.server);
|
||||
cache.set(row.server, outcome);
|
||||
if (outcome.state === 'done' && row.restarted_at) healthy++;
|
||||
if (outcome.restartedNow) restarted++;
|
||||
if (outcome.state === 'pending') held++;
|
||||
if (outcome.state === 'failed') failed++;
|
||||
// Cache last known player count for the dashboard.
|
||||
if (outcome.players !== null) {
|
||||
setServerField(row.server, 'last_players', outcome.players);
|
||||
setServerField(row.server, 'last_checked_at', nowIso());
|
||||
}
|
||||
}
|
||||
const now = nowIso();
|
||||
applyOutcome.run(
|
||||
outcome.state,
|
||||
now,
|
||||
outcome.players,
|
||||
outcome.error,
|
||||
outcome.restartedNow ? 1 : 0,
|
||||
now,
|
||||
row.deployment_id,
|
||||
row.server
|
||||
);
|
||||
emit({
|
||||
type: 'server-state',
|
||||
deploymentId: row.deployment_id,
|
||||
server: row.server,
|
||||
state: outcome.state,
|
||||
players: outcome.players,
|
||||
error: outcome.error
|
||||
});
|
||||
if (!row.files_changed) continue; // A failed file write is not a restart request.
|
||||
const operation = requestRestart(row.server, { kind: 'campaign', id: row.deployment_id });
|
||||
result.checked++;
|
||||
const state = operation.state === 'done' ? 'done'
|
||||
: ['failed','interrupted','cancelled'].includes(operation.state) ? 'failed' : 'pending';
|
||||
const restartedAt = ['awaiting-health','done'].includes(operation.state) ? operation.submitted_at : row.restarted_at;
|
||||
if (restartedAt && !row.restarted_at) result.restarted++;
|
||||
if (state === 'done') result.healthy++;
|
||||
if (state === 'pending') result.held++;
|
||||
if (state === 'failed') result.failed++;
|
||||
const error = state === 'done' ? null : operation.error ?? operation.stage;
|
||||
db().prepare(`UPDATE deployment_servers SET restart_state = ?, checked_at = ?, players = ?, error = ?,
|
||||
attempts = attempts + CASE WHEN ? IS NOT NULL AND checked_at IS NOT ? THEN 1 ELSE 0 END,
|
||||
restarted_at = COALESCE(restarted_at, ?) WHERE deployment_id = ? AND server = ?`)
|
||||
.run(state, operation.checked_at, operation.players, error, operation.checked_at, operation.checked_at, restartedAt, row.deployment_id, row.server);
|
||||
emit({ type: 'server-state', deploymentId: row.deployment_id, server: row.server, state, players: operation.players, error });
|
||||
}
|
||||
|
||||
// Finish deployments where every server row is non-pending/non-failed.
|
||||
db()
|
||||
.prepare(
|
||||
`UPDATE deployments
|
||||
SET finished_at = ?
|
||||
WHERE finished_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM deployment_servers
|
||||
WHERE deployment_id = deployments.id
|
||||
AND restart_state IN ('pending','failed')
|
||||
)`
|
||||
)
|
||||
.run(nowIso());
|
||||
|
||||
const cutoff = new Date(Date.now() - config.deployKeepDays * 24 * 60 * 60 * 1000).toISOString();
|
||||
db()
|
||||
.prepare(
|
||||
`DELETE FROM deployments
|
||||
WHERE started_at < ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM deployment_servers
|
||||
WHERE deployment_id = deployments.id
|
||||
AND restart_state IN ('pending','failed')
|
||||
)`
|
||||
)
|
||||
.run(cutoff);
|
||||
|
||||
db().prepare(`UPDATE deployments SET finished_at = ? WHERE finished_at IS NULL AND NOT EXISTS (
|
||||
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND restart_state IN ('pending','failed','held'))`).run(nowIso());
|
||||
db().prepare(`DELETE FROM deployments WHERE started_at < ? AND NOT EXISTS (
|
||||
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND restart_state IN ('pending','failed','held'))`)
|
||||
.run(new Date(Date.now() - config.deployKeepDays * 86_400_000).toISOString());
|
||||
emit({ type: 'sweep-tick', frozen: false });
|
||||
return { checked, restarted, healthy, held, failed };
|
||||
}
|
||||
|
||||
// Phase 1: wait for the server to be empty, then Komodo-restart it.
|
||||
async function runRestartPhase(server: string): Promise<{ state: 'done' | 'failed' | 'pending'; players: number | null; error: string | null; restartedNow: boolean }> {
|
||||
const waitReason = rolloutWaitReason(server);
|
||||
if (waitReason) return { state: 'pending', players: null, error: waitReason, restartedNow: false };
|
||||
const players = await rconGetPlayerCount(server);
|
||||
if (players === null) {
|
||||
return { state: 'pending', players: null, error: 'rcon unreachable', restartedNow: false };
|
||||
}
|
||||
if (players > 0) {
|
||||
return { state: 'pending', players, error: null, restartedNow: false };
|
||||
}
|
||||
const r = await restartInRollout(server);
|
||||
if (r.kind === 'started') {
|
||||
// Restart triggered — flip restarted_at so the next tick runs the
|
||||
// health phase. Still 'pending' overall until health confirms.
|
||||
return { state: 'pending', players: 0, error: null, restartedNow: true };
|
||||
}
|
||||
return { state: r.kind === 'queued' ? 'pending' : 'failed', players: 0, error: r.message, restartedNow: false };
|
||||
}
|
||||
|
||||
// Phase 2: after Komodo-restart, wait until the container reports running AND
|
||||
// RCON is answering. Only then does the deployment_server row transition to
|
||||
// 'done'. If either check fails we stay pending and try again next tick.
|
||||
function runHealthPhase(server: string): { state: 'done' | 'pending'; players: number | null; error: string | null; restartedNow: boolean } {
|
||||
const h = rolloutHealth(server);
|
||||
if (!h.ok) {
|
||||
return { state: 'pending', players: null, error: h.reason, restartedNow: false };
|
||||
}
|
||||
return { state: 'done', players: h.players, error: null, restartedNow: false };
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { processDestructions, processCreations } from '../fleet/create.js';
|
||||
import { processLifecycle } from '../fleet/lifecycle.js';
|
||||
import { processProvisioning } from '../provisioning/worker.js';
|
||||
import { sweep } from './sweep.js';
|
||||
import { betaTick } from './beta-runner.js';
|
||||
import { emit } from './bus.js';
|
||||
@@ -8,7 +7,7 @@ import { rconStatusResult } from '../integrations/rcon.js';
|
||||
import type { RconFailure } from '../integrations/rcon.js';
|
||||
import { getServer, listServers, setServerField } from '../fleet/servers.js';
|
||||
import { komodoInspectHealthy } from '../integrations/komodo.js';
|
||||
import { restartSettling } from './restart-rollout.js';
|
||||
import { restartSettling } from '../provisioning/index.js';
|
||||
import { classifyHealth } from '../fleet/health.js';
|
||||
import { isProvisioning } from '../fleet/provisioning.js';
|
||||
import { STATUS_REFRESH_MS } from '../../server-freshness.js';
|
||||
@@ -33,25 +32,19 @@ function tick(): Promise<void> {
|
||||
return tickInFlight;
|
||||
}
|
||||
|
||||
async function reconcileRequesters(): Promise<void> {
|
||||
try { await sweep(); } catch (err) { console.error('[worker] campaign error:', err); }
|
||||
try { await betaTick(); } catch (err) { console.error('[worker] beta error:', err); }
|
||||
try { await maintenanceTick(); } catch (err) { console.error('[worker] maintenance error:', err); }
|
||||
}
|
||||
|
||||
async function runTick(): Promise<void> {
|
||||
try { await processCreations(); }
|
||||
catch (err) { console.error('[worker] creation error:', err); }
|
||||
try { await processDestructions(); }
|
||||
catch (err) { console.error('[worker] destruction error:', err); }
|
||||
try { await processLifecycle(); }
|
||||
// Callers submit durable requests first. Exactly one owner admits and runs
|
||||
// lifecycle work, then callers observe the resulting jobs by ID.
|
||||
await reconcileRequesters();
|
||||
try { await processProvisioning(); }
|
||||
catch (err) { console.error('[worker] provisioning error:', err); }
|
||||
try {
|
||||
await sweep();
|
||||
} catch (err) {
|
||||
console.error('[worker] sweep error:', err);
|
||||
}
|
||||
try {
|
||||
await betaTick();
|
||||
} catch (err) {
|
||||
console.error('[worker] beta-runner error:', err);
|
||||
}
|
||||
try { await maintenanceTick(); }
|
||||
catch (err) { console.error('[worker] maintenance error:', err); }
|
||||
await reconcileRequesters();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { db } from '../db/index.js';
|
||||
import { recipeForDeployment, summarizeRecipe } from '../fleet/launch-recipes.js';
|
||||
import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
|
||||
|
||||
export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
|
||||
const id = randomUUID();
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
||||
VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
|
||||
recordDeploymentEvent(id, action === 'create' ? 'Server creation requested'
|
||||
: action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested`);
|
||||
return id;
|
||||
}
|
||||
|
||||
export function deploymentStage(id: string, stage: string): void {
|
||||
const previous = db().prepare('SELECT stage FROM server_deployments WHERE id = ?').get(id) as { stage: string } | undefined;
|
||||
db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id);
|
||||
if (previous && previous.stage !== stage) recordDeploymentEvent(id, stage);
|
||||
}
|
||||
|
||||
export function recordDeploymentEvent(id: string, message: string): void {
|
||||
db().prepare('INSERT INTO server_deployment_events (deployment_id, created_at, message) VALUES (?, ?, ?)')
|
||||
.run(id, new Date().toISOString(), message);
|
||||
}
|
||||
|
||||
export function getDeployment(id: string) {
|
||||
const deployment = db().prepare(`SELECT d.*, j.expected_map, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
|
||||
FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id);
|
||||
if (!deployment) return null;
|
||||
const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id);
|
||||
const sources = db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id').all(id);
|
||||
const launch = recipeForDeployment(id);
|
||||
const collection = db().prepare(`SELECT c.id, c.name FROM (
|
||||
SELECT collection_id, deployment_id FROM server_collection_members
|
||||
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
|
||||
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ?`).get(id) ?? null;
|
||||
return { deployment, events, sources, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
|
||||
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection };
|
||||
}
|
||||
|
||||
// This stage is written only after Komodo has confirmed deployment deletion.
|
||||
export const REMOVING_SERVER = 'Removing server files and registry';
|
||||
|
||||
export function canCancelRemoval(id: string): boolean {
|
||||
return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER);
|
||||
}
|
||||
|
||||
/** Competes atomically with the worker's claim, before any external removal. */
|
||||
export function cancelRemoval(id: string): boolean {
|
||||
return db().transaction(() => {
|
||||
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
|
||||
WHERE id = ? AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(new Date().toISOString(), id, REMOVING_SERVER);
|
||||
if (!result.changes) return false;
|
||||
recordDeploymentEvent(id, 'Queued removal cancelled. The server and its files were kept.');
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
|
||||
export function claimRemoval(id: string): boolean {
|
||||
return db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND action = 'destroy' AND state = 'waiting'").run(id).changes > 0;
|
||||
}
|
||||
|
||||
export function canResumeCleanup(id: string): boolean {
|
||||
return !!db().prepare(`SELECT d.id FROM server_deployments d
|
||||
WHERE d.id = ? AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
||||
.get(id, REMOVING_SERVER);
|
||||
}
|
||||
|
||||
export function resumeCleanup(id: string): boolean {
|
||||
return db().transaction(() => {
|
||||
if (!canResumeCleanup(id)) return false;
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
recordDeploymentEvent(id, 'Resumed local cleanup after confirmed Komodo deletion');
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
|
||||
export function canResumeVerification(id: string): boolean {
|
||||
return !!db().prepare(`SELECT d.id FROM server_deployments d
|
||||
JOIN server_creation_jobs j ON j.deployment_id = d.id
|
||||
JOIN servers s ON s.name = j.server AND s.game_port = j.port
|
||||
WHERE d.id = ? AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
|
||||
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL))) AND d.state IN ('failed', 'interrupted')
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
||||
.get(id);
|
||||
}
|
||||
|
||||
export function canResumeRebuild(id: string): boolean {
|
||||
return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
|
||||
WHERE d.id = ? AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
|
||||
.get(id, REMOVING_SERVER);
|
||||
}
|
||||
|
||||
export function resumeRebuild(id: string): boolean {
|
||||
if (!canResumeRebuild(id)) return false;
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
recordDeploymentEvent(id, 'Resumed rebuild cleanup after confirmed Komodo deletion');
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Resume observation only. Never repeat a container creation or restart. */
|
||||
export function resumeVerification(id: string): boolean {
|
||||
return db().transaction(() => {
|
||||
if (!canResumeVerification(id)) return false;
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
recordDeploymentEvent(id, 'Resumed verification of the existing server');
|
||||
deploymentStage(id, 'Waiting for Komodo health');
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
|
||||
export function finishDeployment(id: string, result: { ok: boolean; message?: string; warnings?: string[] }): void {
|
||||
const state = !result.ok ? 'failed' : result.warnings?.length ? 'warning' : 'done';
|
||||
db().prepare(`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`)
|
||||
.run(state, new Date().toISOString(), result.message ?? null, JSON.stringify(result.warnings ?? []), id);
|
||||
for (const warning of result.warnings ?? []) recordDeploymentEvent(id, warning);
|
||||
recordDeploymentEvent(id, result.ok ? (state === 'warning' ? 'Completed with warnings' : 'Completed') : `Failed: ${result.message ?? 'Unknown error'}`);
|
||||
}
|
||||
|
||||
export function listDeployments() {
|
||||
return db().prepare(`SELECT d.*, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists,
|
||||
(d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval
|
||||
FROM server_deployments d ORDER BY started_at DESC, rowid DESC`).all(REMOVING_SERVER);
|
||||
}
|
||||
|
||||
export function interruptDeployments(): void {
|
||||
const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running'").all() as { id: string }[];
|
||||
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?,
|
||||
error = 'The application restarted before this operation finished. Check the server before retrying.'
|
||||
WHERE state = 'running'`).run(new Date().toISOString());
|
||||
for (const { id } of running) recordDeploymentEvent(id, 'Interrupted by application restart. Check the server before retrying.');
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
// Request/observation boundary used by APIs, campaigns, beta tests, and schedules.
|
||||
// Execution is driven exclusively by the provisioning worker.
|
||||
export { requestLifecycle, requestRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js';
|
||||
export type { LifecycleOperation, RestartSource } from './requests.js';
|
||||
export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextAvailablePort, nextLobbyNumber } from './instances.js';
|
||||
export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js';
|
||||
export { restartSettling, restartRecoveryPending } from './restarts.js';
|
||||
@@ -0,0 +1,796 @@
|
||||
import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
|
||||
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { LaunchPreview } from '$lib/deployment-plan.js';
|
||||
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
|
||||
import { mkdir, chmod, rm, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
|
||||
import { getServer } from '../fleet/servers.js';
|
||||
import { zombieMapBySlug, type GameMode, type ZombieMapEntry, type ZombieGame } from '../fleet/map-aliases.js';
|
||||
import {
|
||||
komodoCreateDeployment,
|
||||
komodoDeploy,
|
||||
komodoDestroyDeployment,
|
||||
komodoDeleteDeployment,
|
||||
komodoInspectHealthy,
|
||||
komodoEnsureTag,
|
||||
komodoUpdateTagsOnResource,
|
||||
type CreateDeploymentSpec,
|
||||
type DeploymentEnvVar
|
||||
} from '../integrations/komodo.js';
|
||||
import { MAP_CONFIG_FILE } from '../fleet/server-config.js';
|
||||
import { recordScript } from '../fleet/inventory.js';
|
||||
import { getVersion } from '../fleet/scripts.js';
|
||||
|
||||
export type CreateServerFailure =
|
||||
| 'bad-request'
|
||||
| 'unconfigured' // Komodo creds / target server missing
|
||||
| 'unknown-map' // slug isn't in the catalog
|
||||
| 'no-slot' // all lobby slots for this map are taken
|
||||
| 'no-port' // port range exhausted
|
||||
| 'fs' // couldn't create host dir
|
||||
| 'komodo' // Komodo CreateDeployment / Deploy rejected
|
||||
| 'db'; // shouldn't happen, but wraps INSERT failures
|
||||
|
||||
export type CreateServerResult =
|
||||
| { ok: true; name: string; port: number; deploymentId: string }
|
||||
| { ok: false; kind: CreateServerFailure; message: string };
|
||||
|
||||
export type DestroyServerResult =
|
||||
| { ok: true; warnings: string[] }
|
||||
| { ok: false; kind: 'not-found' | 'conflict' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] };
|
||||
|
||||
export interface CreateServerInput {
|
||||
game?: ZombieGame; // omitted by older clients: T5
|
||||
gameMode: GameMode; // 'zombie' — only value supported today
|
||||
mapSlug: string; // 'cosmodrome'
|
||||
plutoKey?: string; // saved for this server ID when supplied
|
||||
expectedName?: string; // optional preview guard; never selects a lobby
|
||||
previewId?: string;
|
||||
createdBy?: string; // 'ui' by default
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the lowest lobby number `n` such that `t5_<mode>_<slug>_<n>` isn't in
|
||||
* the servers table. Caps at 20 to stay inside the port range — with 20 total
|
||||
* ports across the whole fleet, no one map can hold more than that.
|
||||
*/
|
||||
export function nextLobbyNumber(baseName: string, additionalNames: string[] = []): number | null {
|
||||
const rows = db()
|
||||
.prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
|
||||
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
|
||||
.all(`${baseName}_%`) as Array<{ name: string }>;
|
||||
const taken = new Set<number>();
|
||||
const prefix = `${baseName}_`;
|
||||
for (const r of [...rows, ...additionalNames.map(name => ({ name }))]) {
|
||||
if (!r.name.startsWith(prefix)) continue;
|
||||
const tail = r.name.slice(prefix.length);
|
||||
if (/^\d+$/.test(tail)) taken.add(Number.parseInt(tail, 10));
|
||||
}
|
||||
for (let n = 1; n <= 20; n++) {
|
||||
if (!taken.has(n)) return n;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lowest unused port in the configured range. Reads every server's game_port
|
||||
* from the DB (which EC controls end-to-end under the new naming scheme, so
|
||||
* the DB is authoritative — no need to ask Komodo).
|
||||
*/
|
||||
export function nextAvailablePort(additionalPorts: number[] = [], releasingServers: string[] = []): number | null {
|
||||
const rows = db()
|
||||
.prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
|
||||
SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
|
||||
.all() as Array<{ name: string; game_port: number }>;
|
||||
const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
|
||||
for (let p = config.portRange.min; p <= config.portRange.max; p++) {
|
||||
if (!taken.has(p)) return p;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function buildEnv(args: {
|
||||
game: ZombieGame;
|
||||
name: string;
|
||||
port: number;
|
||||
plutoKey: string;
|
||||
}): DeploymentEnvVar[] {
|
||||
return [
|
||||
{ variable: 'PLUTAINER_SERVER_NAME', value: args.name },
|
||||
{ variable: 'PLUTAINER_PORT', value: String(args.port) },
|
||||
{ variable: 'PLUTAINER_GAME', value: args.game === 't6' ? 't6zm' : 't5sp' },
|
||||
{ variable: 'PLUTAINER_CONFIG_FILE', value: configFileForGame(args.game) },
|
||||
// Plutainer seeds/loads the main config first. Load our generated map
|
||||
// config afterward, then rotate. A separate exec avoids quoted values
|
||||
// in EXTRA_ARGS, which Plutainer splits on whitespace.
|
||||
{ variable: 'PLUTAINER_MAP_ROTATE', value: 'false' },
|
||||
// Give telemetry scripts the registry name before the first map starts.
|
||||
{ variable: 'PLUTAINER_EXTRA_ARGS', value: `+exec ${MAP_CONFIG_FILE} +set tws_server_id ${args.name} +map_rotate` },
|
||||
{ variable: 'PLUTO_SERVER_KEY', value: args.plutoKey },
|
||||
{ variable: 'PLUTAINER_RCON_PASSWORD', value: config.rcon.password }
|
||||
];
|
||||
}
|
||||
|
||||
function buildSpec(args: {
|
||||
game: ZombieGame;
|
||||
name: string;
|
||||
port: number;
|
||||
hostDir: string;
|
||||
plutoKey: string;
|
||||
recipe: LaunchRecipe;
|
||||
}): CreateDeploymentSpec {
|
||||
return {
|
||||
name: args.name,
|
||||
server: config.komodo.targetServer,
|
||||
image: args.recipe.image,
|
||||
network: config.komodo.defaultNetwork,
|
||||
// Container listens on PLUTAINER_PORT (== args.port), so host and
|
||||
// container ports match — no NAT inside the bridge network.
|
||||
ports: [
|
||||
`${args.port}:${args.port}/udp`,
|
||||
`${args.port}:${args.port}/tcp`
|
||||
],
|
||||
volumes: [
|
||||
`${args.hostDir}:/home/plutainer/app`,
|
||||
`${args.game === 't6' ? config.komodo.t6GameFilesVolume : config.komodo.gameFilesVolume}:/home/plutainer/gamefiles:ro`
|
||||
],
|
||||
environment: buildEnv(args)
|
||||
};
|
||||
}
|
||||
|
||||
let creating = false;
|
||||
interface PreparedCreation {
|
||||
ok: true; name: string; port: number; plutoKey: string; mapEntry: ZombieMapEntry;
|
||||
}
|
||||
|
||||
interface PreviewRow {
|
||||
id: string; expires_at: string; server: string | null; port: number | null;
|
||||
recipe: string; blueprint_name: string | null; deployment_id: string | null;
|
||||
}
|
||||
|
||||
/** Read-only checks; a preview never reserves a slot or writes credentials. */
|
||||
export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown }): LaunchPreview {
|
||||
if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.');
|
||||
let recipe: LaunchRecipe;
|
||||
let blueprintName: string | null = null;
|
||||
if (input.blueprintId !== undefined) {
|
||||
if (typeof input.blueprintId !== 'string') throw new Error('Invalid blueprint.');
|
||||
({ recipe, blueprintName } = blueprintRecipe(input.blueprintId));
|
||||
} else if (input.deploymentId !== undefined) {
|
||||
if (typeof input.deploymentId !== 'string') throw new Error('Invalid deployment.');
|
||||
const source = recipeForDeployment(input.deploymentId);
|
||||
if (!source) throw new Error('This deployment has no recorded launch recipe.');
|
||||
blueprintName = source.blueprintName;
|
||||
// History describes the previous launch; only its map selection is reusable.
|
||||
recipe = captureRecipe(source.recipe.game, source.recipe.mapSlug);
|
||||
} else {
|
||||
if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.');
|
||||
recipe = captureRecipe(input.game, input.mapSlug);
|
||||
}
|
||||
const lobby = nextLobbyNumber(`${recipe.game}_zombie_${recipe.mapSlug}`);
|
||||
const name = lobby === null ? null : `${recipe.game}_zombie_${recipe.mapSlug}_${lobby}`;
|
||||
const port = nextAvailablePort();
|
||||
const configured = !!(config.komodo.address && config.komodo.apiKey && config.komodo.apiSecret && config.komodo.targetServer);
|
||||
const checks: LaunchPreview['checks'] = [
|
||||
{ label: 'Komodo settings', state: configured ? 'pass' : 'blocked', detail: configured ? 'Connection settings and target host are configured.' : 'Configure Komodo credentials and a target host before launching.' },
|
||||
{ label: 'Health verification', state: config.rcon.password ? 'pass' : 'blocked', detail: config.rcon.password ? 'RCON is configured. Container, RCON, and map checks run after launch.' : 'Configure RCON_PASSWORD so the new server can be verified.' },
|
||||
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All 20 lobbies for this map are occupied or reserved.' },
|
||||
{ label: 'Game port', state: port === null ? 'blocked' : 'pass', detail: port === null ? 'No free game port. Free capacity before launching.' : `${port} / UDP and TCP. Availability is checked again at launch.` },
|
||||
{ label: 'Promoted configuration', state: 'pass', detail: recipe.configSource },
|
||||
{ label: 'Game files and host readiness', state: 'info', detail: 'The host must have the selected game and DLC files. This preview does not probe volumes, images, credentials, or network access.' }
|
||||
];
|
||||
const id = randomUUID();
|
||||
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
||||
db().prepare('DELETE FROM server_launch_previews WHERE expires_at < ? AND deployment_id IS NULL').run(new Date().toISOString());
|
||||
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
|
||||
return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName,
|
||||
checks, ready: checks.every(check => check.state !== 'blocked') };
|
||||
}
|
||||
|
||||
// releasingServers is internal to atomic collection queueing; HTTP callers cannot set it.
|
||||
export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult {
|
||||
return db().transaction((): CreateServerResult => {
|
||||
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined;
|
||||
if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' };
|
||||
const recipe: LaunchRecipe | undefined = preview ? JSON.parse(preview.recipe) : undefined;
|
||||
if (recipe && (recipe.game !== (input.game ?? 't5') || recipe.mapSlug !== input.mapSlug || input.gameMode !== 'zombie')) {
|
||||
return { ok: false, kind: 'bad-request', message: 'The selection does not match the reviewed launch recipe.' };
|
||||
}
|
||||
// A repeated submit after a lost response returns the same operation.
|
||||
if (preview?.deployment_id) return { ok: true, name: preview.server!, port: preview.port!, deploymentId: preview.deployment_id };
|
||||
if (preview && preview.expires_at <= new Date().toISOString()) return { ok: false, kind: 'bad-request', message: 'Launch preview expired. Review a fresh preview.' };
|
||||
if (preview && (!preview.server || !preview.port || preview.server !== nextName(input) || preview.port !== nextAvailablePort([], releasingServers))) {
|
||||
return { ok: false, kind: 'no-slot', message: 'Available lobby or port changed. Refresh the preview and review the new server key.' };
|
||||
}
|
||||
if (recipe && !config.rcon.password) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'Health-check settings changed. Refresh the preview.' };
|
||||
}
|
||||
const prepared = prepareCreation(input, releasingServers);
|
||||
if (!prepared.ok) return prepared;
|
||||
// A preview records choices, not version pins. Promotions remain authoritative.
|
||||
const currentRecipe = captureRecipe(input.game ?? 't5', prepared.mapEntry.slug);
|
||||
const id = beginDeployment(prepared.name, 'create', input.createdBy);
|
||||
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, blueprint_name) VALUES (?, ?, ?)')
|
||||
.run(id, JSON.stringify(currentRecipe), preview?.blueprint_name ?? null);
|
||||
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id = ? WHERE id = ?').run(id, preview.id);
|
||||
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical);
|
||||
deploymentStage(id, 'Queued for server creation');
|
||||
return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id };
|
||||
})();
|
||||
}
|
||||
|
||||
/** Reserve the same identity and port before removing any part of the instance. */
|
||||
export function queueRebuildServer(name: string): CreateServerResult {
|
||||
const server = getServer(name);
|
||||
const game = gameOfServer(name);
|
||||
const map = server?.map_slug && zombieMapBySlug(game, server.map_slug);
|
||||
if (!server || !map || !server.game_port || server.komodo_type !== 'deployment' || server.komodo_name !== name
|
||||
|| (server.host_dir && server.host_dir !== serverDir(name))) {
|
||||
return { ok: false, kind: 'bad-request', message: 'Rebuild requires a managed server with its original map, directory, and port.' };
|
||||
}
|
||||
if (!config.komodo.targetServer || !config.rcon.password) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' };
|
||||
if (!getServerKey(name)) return { ok: false, kind: 'bad-request', message: `Save a server key for ${name} in Keys before rebuilding.` };
|
||||
// Validate that the replacement setup can be composed before destruction.
|
||||
const recipe = captureRecipe(game, map.slug);
|
||||
return db().transaction(() => {
|
||||
const id = beginDeployment(name, 'rebuild');
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
|
||||
db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
|
||||
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe));
|
||||
deploymentStage(id, 'Waiting to rebuild the empty server');
|
||||
return { ok: true as const, name, port: server.game_port!, deploymentId: id };
|
||||
})();
|
||||
}
|
||||
|
||||
/** Caller has just verified emptiness and excluded conflicting work. */
|
||||
export async function rebuildServer(id: string): Promise<void> {
|
||||
const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.id = ? AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
|
||||
if (!job) return;
|
||||
if (creating || destroying.size || creationQueue().active.length >= config.createConcurrency) {
|
||||
// The cleanup checkpoint must survive capacity waits and app restarts.
|
||||
if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding');
|
||||
return;
|
||||
}
|
||||
// Recheck prerequisites while the original instance still exists.
|
||||
if (!getServerKey(job.server)) { finishDeployment(id, { ok: false, message: 'Saved server key was removed; rebuild did not begin.' }); return; }
|
||||
try { captureRecipe(gameOfServer(job.server), job.map_slug); }
|
||||
catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); return; }
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
|
||||
destroying.add(job.server);
|
||||
try {
|
||||
const result = job.stage === REMOVING_SERVER
|
||||
? await cleanupDestroyedServer(job.server, id) : await destroyServerUnlocked(job.server, id);
|
||||
if (!result.ok || result.warnings.length) {
|
||||
finishDeployment(id, { ok: false, message: result.ok ? 'Old server files could not be completely removed.' : result.message, warnings: result.warnings });
|
||||
return;
|
||||
}
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE server_rebuild_jobs SET removed_at = ? WHERE deployment_id = ?').run(new Date().toISOString(), id);
|
||||
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Queued for server creation');
|
||||
})();
|
||||
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
|
||||
finally { destroying.delete(job.server); }
|
||||
}
|
||||
|
||||
function nextName(input: CreateServerInput): string | null {
|
||||
const base = `${input.game ?? 't5'}_${input.gameMode}_${input.mapSlug}`;
|
||||
const lobby = nextLobbyNumber(base);
|
||||
return lobby === null ? null : `${base}_${lobby}`;
|
||||
}
|
||||
|
||||
// Internal callers use the same durable queue and capacity limit as HTTP launches.
|
||||
// Await this pass's provisioning when possible; a busy host leaves the job queued.
|
||||
export async function createServer(input: CreateServerInput): Promise<CreateServerResult> {
|
||||
const queued = queueCreateServer(input);
|
||||
if (!queued.ok) return queued;
|
||||
const existing = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(queued.deploymentId) as { state: string };
|
||||
if (existing.state !== 'queued') return queued;
|
||||
const outcomes = await creationPass();
|
||||
const outcome = outcomes.get(queued.deploymentId);
|
||||
if (outcome instanceof Error) throw outcome;
|
||||
return outcome ?? queued;
|
||||
}
|
||||
|
||||
function prepareCreation(input: CreateServerInput, releasingServers: string[] = []): PreparedCreation | Extract<CreateServerResult, { ok: false }> {
|
||||
const game = input.game ?? 't5';
|
||||
if (game !== 't5' && game !== 't6') {
|
||||
return { ok: false, kind: 'bad-request', message: 'Supported games are t5 and t6' };
|
||||
}
|
||||
if (input.gameMode !== 'zombie') {
|
||||
return { ok: false, kind: 'unknown-map', message: `unsupported game mode: ${input.gameMode}` };
|
||||
}
|
||||
if (!config.komodo.address || !config.komodo.apiKey || !config.komodo.apiSecret) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'KOMODO_ADDRESS / KOMODO_API_KEY / KOMODO_API_SECRET must be set' };
|
||||
}
|
||||
if (!config.komodo.targetServer) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'KOMODO_TARGET_SERVER must name the Komodo host that runs new deployments' };
|
||||
}
|
||||
|
||||
const mapEntry = zombieMapBySlug(game, input.mapSlug);
|
||||
if (!mapEntry) {
|
||||
return { ok: false, kind: 'unknown-map', message: `no ${game} zombie map with slug '${input.mapSlug}'` };
|
||||
}
|
||||
|
||||
const baseName = `${game}_${input.gameMode}_${mapEntry.slug}`;
|
||||
const lobby = nextLobbyNumber(baseName);
|
||||
if (lobby === null) {
|
||||
return { ok: false, kind: 'no-slot', message: `all lobby slots (1..20) for ${baseName} are taken` };
|
||||
}
|
||||
const name = `${baseName}_${lobby}`;
|
||||
if (input.expectedName !== undefined && input.expectedName !== name) {
|
||||
return { ok: false, kind: 'no-slot', message: 'Available lobby changed. Review the updated server ID and key, then deploy again.' };
|
||||
}
|
||||
const plutoKey = input.plutoKey?.trim() || getServerKey(name);
|
||||
if (!plutoKey) return { ok: false, kind: 'bad-request', message: `No saved key for ${name}. Enter a Plutonium server key.` };
|
||||
|
||||
const port = nextAvailablePort([], releasingServers);
|
||||
if (port === null) {
|
||||
return {
|
||||
ok: false,
|
||||
kind: 'no-port',
|
||||
message: `no free port in ${config.portRange.min}..${config.portRange.max} — destroy an existing server first`
|
||||
};
|
||||
}
|
||||
|
||||
// Save before external operations so a failed deployment can also be retried.
|
||||
try { saveServerKey(name, plutoKey); }
|
||||
catch (err) { return { ok: false, kind: 'bad-request', message: (err as Error).message }; }
|
||||
|
||||
return { ok: true, name, port, plutoKey, mapEntry };
|
||||
}
|
||||
|
||||
async function createServerUnlocked(input: CreateServerInput, prepared: PreparedCreation, operation: string): Promise<CreateServerResult> {
|
||||
const { name, port, plutoKey, mapEntry } = prepared;
|
||||
const game = gameOfServer(name);
|
||||
// Resolve once more at provisioning so promotions made while queued also apply.
|
||||
// Persist the exact attempted files for history, never as input to future launches.
|
||||
const recipe = captureRecipe(game, mapEntry.slug);
|
||||
db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)
|
||||
ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`)
|
||||
.run(operation, JSON.stringify(recipe));
|
||||
const configFile = recipe.configFile;
|
||||
deploymentStage(operation, 'Preparing server files');
|
||||
const hostDir = serverDir(name);
|
||||
|
||||
// Step 1: filesystem. mkdir mode is masked by umask, so chmod explicitly
|
||||
// after — plutainer expects to be able to write to /home/plutainer/app.
|
||||
try {
|
||||
await mkdir(hostDir, { recursive: true, mode: 0o777 });
|
||||
await chmod(hostDir, 0o777);
|
||||
for (const file of recipe.files) {
|
||||
const dest = destOfKind(name, file.kind);
|
||||
const target = join(dest, file.path);
|
||||
await mkdir(dirname(target), { recursive: true, mode: 0o777 });
|
||||
// Make every newly created ancestor writable by Plutainer, even
|
||||
// when the manager's umask removed write permissions at mkdir.
|
||||
for (let dir = dirname(target); dir !== join(hostDir) && dir !== dirname(dir); dir = dirname(dir)) {
|
||||
await chmod(dir, 0o777);
|
||||
}
|
||||
await writeFile(target, Buffer.from(file.content, 'base64'));
|
||||
await chmod(target, 0o666);
|
||||
}
|
||||
} catch (err) {
|
||||
return { ok: false, kind: 'fs', message: `failed to create ${hostDir}: ${(err as Error).message}` };
|
||||
}
|
||||
|
||||
// Step 2: Komodo CreateDeployment.
|
||||
deploymentStage(operation, 'Creating Komodo deployment');
|
||||
const spec = buildSpec({ game, name, port, hostDir, plutoKey, recipe });
|
||||
const createRes = await komodoCreateDeployment(spec);
|
||||
if (!createRes.ok) {
|
||||
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
|
||||
return { ok: false, kind: 'komodo', message: `CreateDeployment: ${createRes.message}` };
|
||||
}
|
||||
|
||||
// Step 3: DB row — insert *before* execute/Deploy so the sweep sees the
|
||||
// server immediately even if Deploy hangs.
|
||||
try {
|
||||
db().transaction(() => {
|
||||
db().prepare(
|
||||
`INSERT INTO servers
|
||||
(name, komodo_type, komodo_name, game, map,
|
||||
game_mode, map_slug, host_dir, config_file, game_id, image,
|
||||
game_port, created_by, container_state)
|
||||
VALUES
|
||||
(?, 'deployment', ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?,
|
||||
?, ?, 'starting')`
|
||||
).run(
|
||||
name, name, game, mapEntry.canonical,
|
||||
input.gameMode, mapEntry.slug, hostDir,
|
||||
configFile, game === 't6' ? 't6zm' : 't5sp', recipe.image,
|
||||
port, input.createdBy ?? 'ui'
|
||||
);
|
||||
for (const file of recipe.files) {
|
||||
const versionId = file.versionId && getVersion(file.versionId) ? file.versionId : null;
|
||||
recordScript(name, file.path, versionId, file.sha256, file.size, file.path === MAP_CONFIG_FILE ? 'create-map' : 'create');
|
||||
}
|
||||
})();
|
||||
} catch (err) {
|
||||
// Roll back Komodo + FS if the DB insert fails (very unlikely — most
|
||||
// often a race where the same name got inserted twice).
|
||||
await komodoDestroyDeployment(name).catch(() => {});
|
||||
await komodoDeleteDeployment(name).catch(() => {});
|
||||
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
|
||||
return { ok: false, kind: 'db', message: `failed to insert server row: ${(err as Error).message}` };
|
||||
}
|
||||
|
||||
// Step 4: apply Komodo tags so operators can filter the deployment list
|
||||
// by game/map. Best-effort — a tag failure doesn't roll back the server;
|
||||
// the deployment is fully functional without cosmetic tags. Fires in
|
||||
// parallel to keep the create latency down.
|
||||
void applyDeploymentTags(name, game, mapEntry.canonical);
|
||||
|
||||
// Step 5: execute/Deploy — actually start the container.
|
||||
deploymentStage(operation, 'Starting container');
|
||||
const deployRes = await komodoDeploy(name);
|
||||
if (!deployRes.ok) {
|
||||
if (deployRes.kind === 'unreachable' || (deployRes.httpStatus !== undefined && deployRes.httpStatus >= 500)) {
|
||||
// A timeout or server error may arrive after the remote boot began.
|
||||
// Preserve files, identity, and capacity until readiness is known.
|
||||
recordDeploymentEvent(operation, 'Startup response uncertain; verifying the existing server: ' + deployRes.message);
|
||||
return { ok: true, name, port, deploymentId: operation };
|
||||
}
|
||||
// Container never started — undo everything so retrying is clean.
|
||||
await komodoDeleteDeployment(name).catch(() => {});
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
|
||||
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
|
||||
return { ok: false, kind: 'komodo', message: `Deploy: ${deployRes.message}` };
|
||||
}
|
||||
|
||||
return { ok: true, name, port, deploymentId: operation };
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure `game` and `map` tags exist in Komodo, then attach both to the given
|
||||
* deployment. Purely cosmetic — any failure here logs and returns; the caller
|
||||
* treats it as fire-and-forget.
|
||||
*/
|
||||
async function applyDeploymentTags(deployment: string, gameTagName: string, mapTagName: string): Promise<void> {
|
||||
try {
|
||||
const [gameTag, mapTag] = await Promise.all([
|
||||
komodoEnsureTag(gameTagName),
|
||||
komodoEnsureTag(mapTagName)
|
||||
]);
|
||||
const tagIds: string[] = [];
|
||||
if (gameTag.ok) tagIds.push(gameTag.id);
|
||||
else console.warn(`[createServer] tag "${gameTagName}": ${gameTag.message}`);
|
||||
if (mapTag.ok) tagIds.push(mapTag.id);
|
||||
else console.warn(`[createServer] tag "${mapTagName}": ${mapTag.message}`);
|
||||
if (tagIds.length === 0) return;
|
||||
|
||||
const res = await komodoUpdateTagsOnResource('Deployment', deployment, tagIds);
|
||||
if (!res.ok) {
|
||||
console.warn(`[createServer] UpdateResourceMeta (tags) for ${deployment}: ${res.message}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`[createServer] tag application for ${deployment} threw:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
const destroying = new Set<string>();
|
||||
|
||||
// Persist the request before returning so navigation and client disconnects cannot lose it.
|
||||
function hasActiveCreation(name: string): boolean {
|
||||
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name);
|
||||
}
|
||||
|
||||
export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
|
||||
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action IN ('start','restart','stop','rebuild')
|
||||
AND state IN ('waiting','queued','running')`).get(name)) {
|
||||
return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] };
|
||||
}
|
||||
const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
|
||||
if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
|
||||
const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
|
||||
if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id };
|
||||
if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] };
|
||||
const operation = db().transaction(() => {
|
||||
const id = beginDeployment(name, 'destroy', triggeredBy);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting to verify the server is empty');
|
||||
return id;
|
||||
})();
|
||||
return { ok: true, warnings: [], deploymentId: operation };
|
||||
}
|
||||
|
||||
let destructionTick: Promise<void> | null = null;
|
||||
export function processDestructions(): Promise<void> {
|
||||
if (destructionTick) return destructionTick;
|
||||
destructionTick = runDestructions().finally(() => { destructionTick = null; });
|
||||
return destructionTick;
|
||||
}
|
||||
|
||||
async function runDestructions(): Promise<void> {
|
||||
const queued = db().prepare("SELECT id, server, stage FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string; stage: string }[];
|
||||
for (const operation of queued) {
|
||||
if (creating || destroying.has(operation.server)) continue;
|
||||
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
|
||||
destroying.add(operation.server);
|
||||
try {
|
||||
if (operation.stage === REMOVING_SERVER) {
|
||||
const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create'
|
||||
AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id);
|
||||
if (newer) {
|
||||
finishDeployment(operation.id, { ok: false, message: 'A newer creation exists; local cleanup cannot safely resume.' });
|
||||
continue;
|
||||
}
|
||||
if (!claimRemoval(operation.id)) continue;
|
||||
finishDeployment(operation.id, await cleanupDestroyedServer(operation.server, operation.id));
|
||||
continue;
|
||||
}
|
||||
if (!getServer(operation.server)) {
|
||||
if (hasActiveCreation(operation.server)) {
|
||||
deploymentStage(operation.id, 'Waiting for server creation to finish');
|
||||
continue;
|
||||
}
|
||||
finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' });
|
||||
continue;
|
||||
}
|
||||
// A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
|
||||
let players = await rconGetPlayerCount(operation.server);
|
||||
if (players === null) {
|
||||
const health = await komodoInspectHealthy(operation.server);
|
||||
if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
|
||||
}
|
||||
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
|
||||
if (players !== 0) {
|
||||
deploymentStage(operation.id, players === null
|
||||
? 'Waiting for player count: server could not be reached'
|
||||
: 'Waiting for server to empty: ' + players + ' player(s) online');
|
||||
continue;
|
||||
}
|
||||
if (!claimRemoval(operation.id)) continue;
|
||||
deploymentStage(operation.id, 'Server is empty (0 players verified)');
|
||||
const result = await destroyServerUnlocked(operation.server, operation.id);
|
||||
finishDeployment(operation.id, result);
|
||||
} catch (err) {
|
||||
const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string };
|
||||
if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message);
|
||||
else if (row.state !== 'cancelled') finishDeployment(operation.id, { ok: false, message: (err as Error).message });
|
||||
} finally { destroying.delete(operation.server); }
|
||||
}
|
||||
}
|
||||
|
||||
async function destroyServerUnlocked(name: string, operation: string): Promise<DestroyServerResult> {
|
||||
const row = getServer(name);
|
||||
if (!row) {
|
||||
return { ok: false, kind: 'not-found', message: `no server named '${name}'`, warnings: [] };
|
||||
}
|
||||
|
||||
const warnings: string[] = [];
|
||||
|
||||
// 1: stop + remove the container. execute/DestroyDeployment queues the
|
||||
// teardown; the response comes back before the container is actually gone.
|
||||
deploymentStage(operation, 'Stopping and removing container');
|
||||
const destroyRes = await komodoDestroyDeployment(name);
|
||||
if (!destroyRes.ok) {
|
||||
return { ok: false, kind: 'komodo', message: 'DestroyDeployment failed: ' + destroyRes.message, warnings };
|
||||
}
|
||||
|
||||
// 2: wait for the container to actually disappear. Komodo refuses to
|
||||
// DeleteDeployment while a container still exists for it, so poll the
|
||||
// health probe until it stops reporting a running state. `ok:false` here
|
||||
// is the *success* signal — it means the container is no longer inspectable.
|
||||
const pollDeadline = Date.now() + 30_000;
|
||||
while (Date.now() < pollDeadline) {
|
||||
const health = await komodoInspectHealthy(name);
|
||||
if (!health.ok) break;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
}
|
||||
|
||||
// 3: remove the deployment entity itself.
|
||||
deploymentStage(operation, 'Deleting Komodo deployment');
|
||||
const deleteRes = await komodoDeleteDeployment(name);
|
||||
if (!deleteRes.ok) {
|
||||
return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings };
|
||||
}
|
||||
|
||||
return cleanupDestroyedServer(name, operation);
|
||||
}
|
||||
|
||||
async function cleanupDestroyedServer(name: string, operation: string): Promise<DestroyServerResult> {
|
||||
const row = getServer(name);
|
||||
const warnings: string[] = [];
|
||||
// Persist the checkpoint before local work so failed cleanup can resume without RCON.
|
||||
// Legacy operations already recorded this same stage after Komodo deletion.
|
||||
deploymentStage(operation, REMOVING_SERVER);
|
||||
// Host directory. Fall back to derived path if host_dir wasn't set on
|
||||
// legacy rows.
|
||||
const hostDir = row?.host_dir ?? serverDir(name);
|
||||
try {
|
||||
await rm(hostDir, { recursive: true, force: true });
|
||||
} catch (err) {
|
||||
warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
// Remove server-owned beta and deployment rows before the registry row.
|
||||
// Test evidence cascades from runs; inventory and snapshots cascade from servers.
|
||||
try {
|
||||
const tx = db().transaction((n: string) => {
|
||||
db().prepare('DELETE FROM deployment_servers WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM deployment_files WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_servers WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_server_scripts WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_test_runs WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_test_coverage WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM beta_file_changes WHERE server = ?').run(n);
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(n);
|
||||
});
|
||||
tx(name);
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
kind: 'db',
|
||||
message: `failed to remove DB rows for ${name}: ${(err as Error).message}`,
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
const unfinished = db().prepare("SELECT id, action FROM server_deployments WHERE server = ? AND state = 'awaiting-health'").all(name) as { id: string; action: string }[];
|
||||
for (const { id, action } of unfinished) finishDeployment(id, { ok: false, message: action === 'create'
|
||||
? 'Server destroyed before creation health checks completed' : 'Server destroyed before operation verification completed' });
|
||||
return { ok: true, warnings };
|
||||
}
|
||||
interface CreationJob {
|
||||
deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
|
||||
}
|
||||
|
||||
async function provisionCreation(id: string): Promise<CreateServerResult> {
|
||||
const job = db().prepare('SELECT j.*, d.triggered_by FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE j.deployment_id = ?').get(id) as CreationJob;
|
||||
creating = true;
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(id);
|
||||
try {
|
||||
const mapEntry = zombieMapBySlug(gameOfServer(job.server), job.map_slug);
|
||||
const plutoKey = getServerKey(job.server);
|
||||
if (!mapEntry || !plutoKey) {
|
||||
const result = { ok: false as const, kind: 'bad-request' as const, message: 'Saved server key or selected map is no longer available' };
|
||||
finishDeployment(id, result);
|
||||
return result;
|
||||
}
|
||||
const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by },
|
||||
{ ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
|
||||
if (!result.ok) finishDeployment(id, result);
|
||||
else {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for Komodo health');
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
const server = getServer(job.server);
|
||||
if (server?.game_port === job.port) {
|
||||
// A lost response may have happened after Komodo started the server.
|
||||
// Keep its slot and observe readiness instead of starting another boot.
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message);
|
||||
deploymentStage(id, 'Verifying existing server after interrupted startup');
|
||||
} else finishDeployment(id, { ok: false, message: (err as Error).message });
|
||||
throw err;
|
||||
} finally { creating = false; }
|
||||
}
|
||||
|
||||
type CreationOutcomes = Map<string, CreateServerResult | Error>;
|
||||
let creationTick: Promise<CreationOutcomes> | null = null;
|
||||
function creationPass(): Promise<CreationOutcomes> {
|
||||
if (creationTick) return creationTick;
|
||||
creationTick = runCreations().finally(() => { creationTick = null; });
|
||||
return creationTick;
|
||||
}
|
||||
|
||||
export async function processCreations(): Promise<void> {
|
||||
await creationPass();
|
||||
}
|
||||
|
||||
/** One host-wide FIFO, shared by collections and individual launches. */
|
||||
export function creationQueue() {
|
||||
type Entry = { id: string; server: string; state: string; stage: string };
|
||||
const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments
|
||||
WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
|
||||
ORDER BY started_at, rowid`).all() as Entry[];
|
||||
return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
|
||||
}
|
||||
|
||||
function creationBlocker(id: string): string | null {
|
||||
const dependency = db().prepare(`SELECT d.server, d.state FROM server_creation_dependencies dep
|
||||
JOIN server_deployments d ON d.id = dep.removal_id
|
||||
WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { server: string; state: string } | undefined;
|
||||
if (dependency) return `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`;
|
||||
const occupied = db().prepare(`SELECT s.name FROM servers s JOIN server_creation_jobs j ON s.game_port = j.port
|
||||
WHERE j.deployment_id = ? LIMIT 1`).get(id) as { name: string } | undefined;
|
||||
return occupied ? `Waiting for port to be released by ${occupied.name}` : null;
|
||||
}
|
||||
|
||||
async function runCreations(): Promise<CreationOutcomes> {
|
||||
const outcomes: CreationOutcomes = new Map();
|
||||
if (creating || destroying.size > 0) return outcomes;
|
||||
// Restart recovery is observation-only: never repeat external creation or
|
||||
// deployment. A known interrupted startup must retain its readiness gate.
|
||||
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
|
||||
for (const { id } of interrupted) {
|
||||
if (resumeVerification(id)) continue;
|
||||
// A queued removal does not make the interrupted container disappear.
|
||||
// Keep its slot until removal finishes, without resuming normal work or
|
||||
// confusing it with a newer creation of the same server identity.
|
||||
const removing = db().prepare(`SELECT 1 FROM server_deployments d
|
||||
JOIN server_creation_jobs j ON j.deployment_id = d.id
|
||||
JOIN servers s ON s.name = j.server AND s.game_port = j.port
|
||||
WHERE d.id = ? AND EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server
|
||||
AND newer.rowid > d.rowid AND newer.action = 'destroy' AND newer.state IN ('waiting', 'running'))
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server
|
||||
AND newer.rowid > d.rowid AND newer.action = 'create')`).get(id);
|
||||
if (removing) {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for requested server removal after interrupted startup');
|
||||
}
|
||||
}
|
||||
|
||||
// Check only previously started servers. Newly launched servers keep their
|
||||
// slot until a later pass, even if a health probe would answer immediately.
|
||||
const checking = db().prepare(`SELECT j.* FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[];
|
||||
for (const job of checking) {
|
||||
const id = job.deployment_id;
|
||||
if (getServer(job.server)?.game_port !== job.port) {
|
||||
finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' });
|
||||
continue;
|
||||
}
|
||||
if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
|
||||
try {
|
||||
const komodo = await komodoInspectHealthy(job.server);
|
||||
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
|
||||
const rcon = await rconStatusResult(job.server);
|
||||
if (!rcon.ok) { deploymentStage(id, 'Waiting for RCON health: ' + rcon.reason); continue; }
|
||||
const actualMap = rcon.status.map;
|
||||
if (actualMap !== job.expected_map) {
|
||||
deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', RCON reports ' + (actualMap ?? 'no map'));
|
||||
continue;
|
||||
}
|
||||
deploymentStage(id, 'Verified Komodo healthy, RCON healthy, and map ' + actualMap);
|
||||
finishDeployment(id, { ok: true });
|
||||
} catch (err) {
|
||||
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot the budget once. Fast success/failure must not turn a single
|
||||
// worker pass into an unbounded stream of expensive container startups.
|
||||
const queue = creationQueue();
|
||||
const available = Math.max(0, config.createConcurrency - queue.active.length);
|
||||
const eligible = queue.queued.filter(({ id }) => {
|
||||
const blocker = creationBlocker(id);
|
||||
if (blocker) deploymentStage(id, blocker);
|
||||
return !blocker;
|
||||
});
|
||||
for (const { id } of eligible.slice(0, available)) {
|
||||
if (creating || destroying.size > 0) break;
|
||||
// API calls can cancel a selected job or resume an older startup while
|
||||
// a previous provision awaits Komodo. Recheck without expanding this pass.
|
||||
if (creationQueue().active.length >= config.createConcurrency) break;
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
|
||||
const blocker = creationBlocker(id);
|
||||
if (blocker) {
|
||||
deploymentStage(id, blocker);
|
||||
continue;
|
||||
}
|
||||
try { outcomes.set(id, await provisionCreation(id)); }
|
||||
catch (err) {
|
||||
const failure = err instanceof Error ? err : new Error(String(err));
|
||||
outcomes.set(id, failure);
|
||||
console.error('[creation] provisioning failed:', failure);
|
||||
}
|
||||
}
|
||||
return outcomes;
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { getServer } from '../fleet/servers.js';
|
||||
import { beginDeployment, deploymentStage, recordDeploymentEvent } from './history.js';
|
||||
import { queueRebuildServer } from './instances.js';
|
||||
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
|
||||
|
||||
export interface LifecycleOperation {
|
||||
id: string; server: string; action: string; state: string; stage: string;
|
||||
submitted_at: string | null; finished_at: string | null; checked_at: string | null;
|
||||
players: number | null; error: string | null;
|
||||
}
|
||||
export interface RestartSource { kind: 'campaign' | 'beta' | 'maintenance'; id: string }
|
||||
export function provisioningOperation(id: string): LifecycleOperation | undefined {
|
||||
return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id) as LifecycleOperation | undefined;
|
||||
}
|
||||
|
||||
const sourceTables = {
|
||||
campaign: { table: 'deployment_servers', key: 'deployment_id' },
|
||||
beta: { table: 'beta_servers', key: 'beta_id' },
|
||||
maintenance: { table: 'maintenance_jobs', key: 'id' }
|
||||
} as const;
|
||||
|
||||
/** Submit once per caller attempt; a lost response or terminal failure never replays a restart. */
|
||||
export function requestRestart(server: string, source: RestartSource): LifecycleOperation {
|
||||
return db().transaction(() => {
|
||||
const { table, key } = sourceTables[source.kind];
|
||||
const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ?`)
|
||||
.get(source.id, server) as { provisioning_id: string | null; restarted_at: string | null } | undefined;
|
||||
if (!row) throw new Error('Restart source no longer exists');
|
||||
if (row.provisioning_id) {
|
||||
const operation = provisioningOperation(row.provisioning_id);
|
||||
if (!operation) throw new Error('Provisioning job is missing; inspect the server before retrying');
|
||||
if (row.restarted_at && !operation.submitted_at && operation.state === 'waiting') {
|
||||
adoptSubmittedRestart(operation.id, server, row.restarted_at);
|
||||
return provisioningOperation(operation.id)!;
|
||||
}
|
||||
return operation;
|
||||
}
|
||||
let id: string | undefined;
|
||||
if (row.restarted_at) {
|
||||
// The previous release already recorded some submissions in history.
|
||||
// Bind to that exact interval, never to a newer restart of the server.
|
||||
id = (db().prepare(`SELECT d.id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id
|
||||
WHERE r.server = ? AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`)
|
||||
.get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id;
|
||||
if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id);
|
||||
}
|
||||
// Only not-yet-submitted campaign changes can share an activation. A later
|
||||
// file deployment must never be credited to an already running restart.
|
||||
if (source.kind === 'campaign' && !row.restarted_at) {
|
||||
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.action = 'restart'
|
||||
AND d.state = 'waiting' AND d.submitted_at IS NULL
|
||||
AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign')
|
||||
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign')
|
||||
ORDER BY d.rowid LIMIT 1`).get(server) as { id: string } | undefined)?.id;
|
||||
}
|
||||
if (!id) {
|
||||
id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting to verify the server is empty');
|
||||
if (row.restarted_at) adoptSubmittedRestart(id, server, row.restarted_at);
|
||||
}
|
||||
db().prepare('INSERT OR IGNORE INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, ?, ?, ?)')
|
||||
.run(id, source.kind, source.id, server);
|
||||
db().prepare(`UPDATE ${table} SET provisioning_id = ? WHERE ${key} = ? AND server = ?`).run(id, source.id, server);
|
||||
return provisioningOperation(id)!;
|
||||
})();
|
||||
}
|
||||
|
||||
/** Upgrade observation only: existing restart intent is never resubmitted. */
|
||||
function adoptSubmittedRestart(id: string, server: string, submittedAt: string): void {
|
||||
const slot = db().prepare('SELECT operation_id, requested_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as
|
||||
{ operation_id: string | null; requested_at: string; healthy_at: string | null; players: number | null } | undefined;
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', submitted_at = ? WHERE id = ?").run(submittedAt, id);
|
||||
if (slot && ((slot.operation_id && slot.requested_at >= submittedAt) || (slot.requested_at < submittedAt && !slot.healthy_at))) {
|
||||
db().prepare("UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE id = ?")
|
||||
.run(new Date().toISOString(), 'Existing restart cannot be correlated safely. Inspect the server before retrying.', id);
|
||||
return;
|
||||
}
|
||||
if (slot && slot.requested_at >= submittedAt) {
|
||||
db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ?').run(id, server);
|
||||
} else {
|
||||
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
|
||||
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at, operation_id = excluded.operation_id,
|
||||
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`).run(server, submittedAt, id);
|
||||
}
|
||||
if (slot?.healthy_at && slot.requested_at >= submittedAt) {
|
||||
db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ?, players = ? WHERE id = ?").run(slot.healthy_at, slot.players, id);
|
||||
}
|
||||
deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again');
|
||||
}
|
||||
|
||||
export function cancelProvisioning(id: string, reason = 'Cancelled by requester'): boolean {
|
||||
return db().transaction(() => {
|
||||
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL
|
||||
WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL`)
|
||||
.run(new Date().toISOString(), reason, id);
|
||||
if (result.changes) recordDeploymentEvent(id, reason);
|
||||
return result.changes > 0;
|
||||
})();
|
||||
}
|
||||
|
||||
/** Explicit caller retry starts a new job and keeps the old job as history. */
|
||||
export function releaseRestartRequest(server: string, source: RestartSource): void {
|
||||
const { table, key } = sourceTables[source.kind];
|
||||
const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ?`).get(source.id, server) as { provisioning_id: string | null } | undefined;
|
||||
if (row?.provisioning_id) cancelProvisioning(row.provisioning_id, 'Superseded by a fresh caller attempt');
|
||||
db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ?`).run(source.id, server);
|
||||
}
|
||||
|
||||
export function requestLifecycle(server: string, action: RuntimeAction) {
|
||||
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
|
||||
if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' };
|
||||
return db().transaction(() => {
|
||||
const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
|
||||
AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
|
||||
if (existing) return existing.action === action
|
||||
? { ok: true as const, deploymentId: existing.id }
|
||||
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
|
||||
const row = getServer(server);
|
||||
if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' };
|
||||
if (action === 'rebuild') {
|
||||
const result = queueRebuildServer(server);
|
||||
return result.ok ? { ok: true as const, deploymentId: result.deploymentId }
|
||||
: { ok: false as const, status: 400, error: result.message };
|
||||
}
|
||||
const id = beginDeployment(server, action);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty');
|
||||
return { ok: true as const, deploymentId: id };
|
||||
})();
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
import { config } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
|
||||
import { rconStatusResult } from '../integrations/rcon.js';
|
||||
import { deploymentStage, finishDeployment } from './history.js';
|
||||
|
||||
// Komodo queues restart work. Don't mistake the still-running old container
|
||||
// for a completed restart when an API kick immediately follows the request.
|
||||
const HEALTH_POLL_MS = 15_000;
|
||||
const requestsInFlight = new Set<string>();
|
||||
|
||||
interface RestartSlot {
|
||||
server: string;
|
||||
requested_at: string;
|
||||
checked_at: string | null;
|
||||
healthy_at: string | null;
|
||||
players: number | null;
|
||||
error: string | null;
|
||||
operation_id: string | null;
|
||||
}
|
||||
|
||||
function slotFor(server: string): RestartSlot | undefined {
|
||||
return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
|
||||
}
|
||||
|
||||
export function restartRecoveryPending(server: string): boolean {
|
||||
return slotFor(server)?.healthy_at === null;
|
||||
}
|
||||
|
||||
/** The queued restart may still expose the previous healthy container. */
|
||||
export function restartSettling(server: string): boolean {
|
||||
const slot = slotFor(server);
|
||||
return requestsInFlight.has(server) || !!(slot && !slot.healthy_at
|
||||
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
|
||||
}
|
||||
|
||||
export function rolloutWaitReason(server: string, operationId = ''): string | null {
|
||||
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ?
|
||||
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
|
||||
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) {
|
||||
return 'Waiting for the current provisioning operation';
|
||||
}
|
||||
const slot = slotFor(server);
|
||||
if (slot?.healthy_at === null) {
|
||||
return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
|
||||
}
|
||||
const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
|
||||
return n >= config.restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null;
|
||||
}
|
||||
|
||||
export type RolloutRestartResult =
|
||||
| { kind: 'started' }
|
||||
| { kind: 'queued'; message: string }
|
||||
| { kind: 'failed'; message: string };
|
||||
|
||||
/** Reserve capacity before any async I/O; callers still check for players first. */
|
||||
export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise<RolloutRestartResult> {
|
||||
const operationId = options.operationId;
|
||||
const waitReason = db().transaction(() => {
|
||||
const reason = rolloutWaitReason(server, operationId);
|
||||
if (reason) return reason;
|
||||
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
|
||||
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
|
||||
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
|
||||
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
|
||||
.run(server, new Date().toISOString(), operationId);
|
||||
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container');
|
||||
return null;
|
||||
}).immediate();
|
||||
if (waitReason) return { kind: 'queued', message: waitReason };
|
||||
|
||||
requestsInFlight.add(server);
|
||||
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
|
||||
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
|
||||
try {
|
||||
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
|
||||
if (result.ok) {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(operationId);
|
||||
deploymentStage(operationId!, 'Waiting for container and RCON health');
|
||||
return { kind: 'started' };
|
||||
}
|
||||
// These failures cannot have queued a remote restart. Other failures
|
||||
// (especially a timeout) may have: retain capacity until recovery.
|
||||
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
|
||||
db().prepare('DELETE FROM restart_rollout WHERE server = ?').run(server);
|
||||
} else {
|
||||
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server);
|
||||
}
|
||||
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
|
||||
return { kind: 'failed', message: `komodo: ${result.message}` };
|
||||
} catch (err) {
|
||||
const message = `komodo: ${(err as Error).message}`;
|
||||
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server);
|
||||
finishDeployment(operationId!, { ok: false, message });
|
||||
return { kind: 'failed', message };
|
||||
} finally {
|
||||
// Start the grace period at completion, including slow/failed requests.
|
||||
db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ?')
|
||||
.run(new Date().toISOString(), server);
|
||||
requestsInFlight.delete(server);
|
||||
}
|
||||
}
|
||||
|
||||
let refreshInFlight: Promise<void> | null = null;
|
||||
|
||||
/** Poll all occupied slots, even if their originating beta was already closed. */
|
||||
export function refreshRestartRollout(): Promise<void> {
|
||||
if (refreshInFlight) return refreshInFlight;
|
||||
refreshInFlight = refresh().finally(() => { refreshInFlight = null; });
|
||||
return refreshInFlight;
|
||||
}
|
||||
|
||||
async function refresh(): Promise<void> {
|
||||
const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server')
|
||||
.all() as RestartSlot[];
|
||||
for (const slot of slots) {
|
||||
if (requestsInFlight.has(slot.server)) continue;
|
||||
if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
|
||||
let error: string | null = null;
|
||||
let players: number | null = null;
|
||||
try {
|
||||
const health = await komodoInspectHealthy(slot.server);
|
||||
if (!health.ok) {
|
||||
error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
|
||||
} else {
|
||||
const rcon = await rconStatusResult(slot.server);
|
||||
if (!rcon.ok) error = `rcon not answering: ${rcon.reason}`;
|
||||
else players = rcon.status.players;
|
||||
}
|
||||
} catch (err) {
|
||||
error = `health check: ${(err as Error).message}`;
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
|
||||
WHERE server = ? AND requested_at = ? AND healthy_at IS NULL`)
|
||||
.run(now, error ? null : now, players, error, slot.server, slot.requested_at);
|
||||
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'awaiting-health'").get(slot.operation_id)) {
|
||||
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
|
||||
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
|
||||
else {
|
||||
deploymentStage(slot.operation_id, 'Verified container and RCON health');
|
||||
finishDeployment(slot.operation_id, { ok: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } {
|
||||
const slot = slotFor(server);
|
||||
if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 };
|
||||
return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' };
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { getServer } from '../fleet/servers.js';
|
||||
import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
|
||||
import { processCreations, processDestructions, rebuildServer } from './instances.js';
|
||||
import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
|
||||
import { rconGetPlayerCount } from '../integrations/rcon.js';
|
||||
import { refreshRestartRollout, restartInRollout, rolloutWaitReason } from './restarts.js';
|
||||
import { cancelProvisioning } from './requests.js';
|
||||
import type { RuntimeAction } from '../../provisioning.js';
|
||||
|
||||
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
|
||||
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
|
||||
|
||||
interface Source { kind: 'campaign' | 'beta' | 'maintenance'; source_id: string }
|
||||
const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?').all(id) as Source[];
|
||||
const priority = (sources: Source[]) => sources.some(s => s.kind === 'beta') ? 0 : sources.some(s => s.kind === 'campaign') ? 1 : sources.some(s => s.kind === 'maintenance') ? 3 : 2;
|
||||
|
||||
/** Source work may hold or withdraw a request, but never executes it. */
|
||||
function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled' {
|
||||
const sources = sourcesFor(operation.id);
|
||||
if (!sources.length) return 'ready';
|
||||
let held = false;
|
||||
for (const source of sources) {
|
||||
if (source.kind === 'campaign') {
|
||||
const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ?')
|
||||
.get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined;
|
||||
if (row?.provisioning_id !== operation.id || !row.files_changed) continue;
|
||||
if (['pending','failed'].includes(row.restart_state)) return 'ready';
|
||||
if (['held','manual'].includes(row.restart_state)) held = true;
|
||||
} else if (source.kind === 'beta') {
|
||||
const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
|
||||
LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ?`)
|
||||
.get(source.source_id, operation.server) as { state: string; stage: string; provisioning_id: string; outcome: string | null } | undefined;
|
||||
if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue;
|
||||
if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue;
|
||||
if (!row.outcome || ['active','passed'].includes(row.outcome)) return 'ready';
|
||||
if (row.outcome === 'paused') held = true;
|
||||
} else {
|
||||
const row = db().prepare('SELECT state, provisioning_id FROM maintenance_jobs WHERE id = ?').get(source.source_id) as { state: string; provisioning_id: string } | undefined;
|
||||
if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) return 'ready';
|
||||
}
|
||||
}
|
||||
return held ? 'held' : 'cancelled';
|
||||
}
|
||||
|
||||
function blocker(server: string, id: string): string | null {
|
||||
const sources = sourcesFor(id);
|
||||
const rank = priority(sources);
|
||||
const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ?
|
||||
AND state IN ('queued','waiting','running','awaiting-health')`).all(server, id) as { id: string; state: string; action: string; rowid: number }[];
|
||||
const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { rowid: number };
|
||||
if (others.some(other => other.state !== 'waiting' || other.action === 'destroy'
|
||||
|| priority(sourcesFor(other.id)) < rank || (priority(sourcesFor(other.id)) === rank && other.rowid < own.rowid))) {
|
||||
return 'Waiting for another provisioning operation on this server';
|
||||
}
|
||||
// Beta activation must not wait on the campaign rows that the beta itself
|
||||
// holds. Campaign/maintenance callers likewise never block on their own job.
|
||||
if (rank > 0 && db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
|
||||
WHERE bs.server = ? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server';
|
||||
if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ?
|
||||
AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS (
|
||||
SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health')))
|
||||
LIMIT 1`).get(server)) return 'Waiting for campaign work on this server';
|
||||
return rolloutWaitReason(server, id);
|
||||
}
|
||||
|
||||
interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
|
||||
let ticking: Promise<void> | null = null;
|
||||
export function processProvisioning(): Promise<void> {
|
||||
if (ticking) return ticking;
|
||||
ticking = run().finally(() => { ticking = null; });
|
||||
return ticking;
|
||||
}
|
||||
|
||||
export const processLifecycle = processProvisioning;
|
||||
|
||||
async function run(): Promise<void> {
|
||||
await processCreations();
|
||||
await processDestructions();
|
||||
await refreshRestartRollout();
|
||||
const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments
|
||||
WHERE action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
|
||||
ORDER BY started_at, rowid`).all() as Operation[];
|
||||
for (const operation of operations) {
|
||||
try { await processOperation(operation); }
|
||||
catch (err) {
|
||||
// Observation failures are retried; remote mutations are never replayed.
|
||||
deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function processOperation(operation: Operation): Promise<void> {
|
||||
const { id, server, action } = operation;
|
||||
if (operation.state === 'waiting') {
|
||||
const eligibility = sourceEligibility(operation);
|
||||
if (eligibility === 'cancelled') { cancelProvisioning(id, 'Requester cancelled or replaced this operation'); return; }
|
||||
if (eligibility === 'held') { deploymentStage(id, 'Waiting for the requester to resume'); return; }
|
||||
}
|
||||
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
|
||||
if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
|
||||
if (!komodoConfigured() || !rconConfigured()) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; }
|
||||
if (operation.state === 'awaiting-health') {
|
||||
const health = await komodoInspectHealthy(server);
|
||||
if (isStopped(health.state)) {
|
||||
db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
|
||||
deploymentStage(id, 'Verified container stopped');
|
||||
finishDeployment(id, { ok: true });
|
||||
} else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
|
||||
return;
|
||||
}
|
||||
const reason = blocker(server, id);
|
||||
if (reason) { deploymentStage(id, reason); return; }
|
||||
const health = await komodoInspectHealthy(server);
|
||||
if (action !== 'start' && !isStopped(health.state)) {
|
||||
const players = await rconGetPlayerCount(server);
|
||||
db().prepare('UPDATE server_deployments SET players = ?, checked_at = ? WHERE id = ?').run(players, new Date().toISOString(), id);
|
||||
if (players !== 0) {
|
||||
deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Requests can arrive during the probes. Recheck before claiming the action.
|
||||
if (blocker(server, id) || sourceEligibility(operation) !== 'ready') return;
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'waiting'").get(id)) return;
|
||||
if (action === 'rebuild') { await rebuildServer(id); return; }
|
||||
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
|
||||
if (!claim.changes) return;
|
||||
if (action === 'start' || action === 'restart') {
|
||||
const result = await restartInRollout(server, { operationId: id, action });
|
||||
if (result.kind === 'queued') {
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
deploymentStage(id, result.message);
|
||||
}
|
||||
return;
|
||||
}
|
||||
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id);
|
||||
deploymentStage(id, 'Stopping container');
|
||||
// Persist intent first; a lost reply must not issue another stop later.
|
||||
try {
|
||||
const result = await komodoRuntimeAction(server, 'stop');
|
||||
if (!result.ok) { finishDeployment(id, { ok: false, message: result.message }); return; }
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for container to stop');
|
||||
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
|
||||
}
|
||||
@@ -95,6 +95,7 @@
|
||||
<strong>{job.server}</strong> · {job.state} · {$timeFormat.dateTime(job.requested_at)}
|
||||
{#if job.target_revision} · Target r{job.target_revision}{/if}
|
||||
{#if job.detail}<small>{job.detail}</small>{/if}
|
||||
{#if job.provisioning_id}<a href="/provisioning/{encodeURIComponent(job.provisioning_id)}">View provisioning</a>{/if}
|
||||
{#if job.state === 'queued'}<Button size="sm" disabled={busy} onclick={() => refresh({ action: 'cancel', id: job.id })}>Cancel</Button>{/if}
|
||||
</li>{/each}</ul>
|
||||
{/if}
|
||||
|
||||
Reference in New Issue
Block a user