josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Complete provisioning job detail pages and lifecycle evidence

Browse Source
This commit is contained in:
josh
2026-09-28 14:17:19 -04:00
parent 5b65520bbc
commit 3fb59cd598
41 changed files with 2866 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/instances.ts
+182 -52
View File
@@ -1,5 +1,5 @@
import { fleetSettings } from '../settings.js';
import { probePlayerCount, probeServerStatus } from '../fleet/status-evidence.js';
import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence.js';
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
import { randomUUID } from 'node:crypto';
import type { LaunchPreview } from '$lib/deployment-plan.js';
@@ -25,6 +25,8 @@ import {
import { MAP_CONFIG_FILE } from '../fleet/server-config.js';
import { recordScript } from '../fleet/inventory.js';
import { getVersion } from '../fleet/scripts.js';
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
import { recordProgress, type ProgressUpdate } from './progress.js';
export type CreateServerFailure =
| 'bad-request'
@@ -147,6 +149,13 @@ function buildSpec(args: {
}
let creating = false;
async function observeCommand<T>(operation: string, command: () => Promise<T>): Promise<T> {
try { return await command(); }
catch (err) {
recordProgress(operation, { command: { state: 'uncertain', message: 'No confirmed command result was received: ' + (err as Error).message } });
throw err;
}
}
interface PreparedCreation {
ok: true; name: string; port: number; plutoKey: string; mapEntry: ZombieMapEntry;
}
@@ -225,7 +234,8 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
.run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical);
deploymentStage(id, 'Queued for server creation');
deploymentStage(id, 'Queued for server creation', { phase: 'startup-queue', recipeStatus: 'queued', observedAt: null,
blocker: { code: 'startup-queue', message: 'Waiting for startup admission and available capacity.', automatic: true } });
return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id };
})();
}
@@ -252,7 +262,8 @@ export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServ
.run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe));
deploymentStage(id, 'Waiting to rebuild the empty server');
deploymentStage(id, 'Waiting to rebuild the empty server', { phase: 'eligibility', recipeStatus: 'queued', observedAt: null,
blocker: { code: 'eligibility', message: 'Waiting for safe rebuild eligibility checks.', automatic: true } });
return { ok: true as const, name, port: server.game_port!, deploymentId: id };
})();
}
@@ -266,12 +277,15 @@ export async function rebuildServer(id: string): Promise<void> {
if (blocker) {
// Keep the cleanup checkpoint intact when resuming after remote deletion.
if (job.stage === REMOVING_SERVER) finishDeployment(id, { ok: false, message: blocker });
else deploymentStage(id, blocker);
else deploymentStage(id, blocker, { phase: 'eligibility', blocker: betaRemovalEvidence(job.server, blocker), checks: [] });
return;
}
if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) {
// The cleanup checkpoint must survive capacity waits and app restarts.
if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding');
const progress: ProgressUpdate = { phase: job.stage === REMOVING_SERVER ? 'cleanup' : 'eligibility',
blocker: { code: 'startup-capacity', message: 'Waiting for a server creation slot before rebuilding', automatic: true } };
if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding', progress);
else recordProgress(id, progress);
return;
}
// Recheck prerequisites while the original instance still exists.
@@ -290,7 +304,9 @@ export async function rebuildServer(id: string): Promise<void> {
db().transaction(() => {
db().prepare('UPDATE server_rebuild_jobs SET removed_at = ? WHERE deployment_id = ?').run(new Date().toISOString(), id);
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
deploymentStage(id, 'Queued for server creation');
deploymentStage(id, 'Queued for server creation', { phase: 'startup-queue', checks: [],
blocker: { code: 'startup-queue', message: 'Old instance removed; waiting for fresh-instance startup admission.', automatic: true },
checkpoint: { key: 'cleanup', label: 'Old instance files and registry removed' } });
})();
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
finally { destroying.delete(job.server); }
@@ -369,11 +385,15 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
// Resolve once more at provisioning so promotions made while queued also apply.
// Persist the exact attempted files for history, never as input to future launches.
const recipe = captureRecipe(game, mapEntry.slug);
db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)
ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`)
.run(operation, JSON.stringify(recipe));
db().transaction(() => {
db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)
ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`)
.run(operation, JSON.stringify(recipe));
deploymentStage(operation, 'Preparing server files', { phase: 'prepare', recipeStatus: 'attempted', blocker: null, checks: [],
checkpoint: { key: 'startup-queue', label: 'Admitted to startup' },
event: { kind: 'recipe', message: 'Promoted files selected for this creation attempt.', details: { capturedAt: recipe.capturedAt } } });
})();
const configFile = recipe.configFile;
deploymentStage(operation, 'Preparing server files');
const hostDir = serverDir(name);
// Step 1: filesystem. mkdir mode is masked by umask, so chmod explicitly
@@ -398,13 +418,18 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
}
// Step 2: Komodo CreateDeployment.
deploymentStage(operation, 'Creating Komodo deployment');
deploymentStage(operation, 'Creating Komodo deployment', { phase: 'create-deployment',
checkpoint: { key: 'prepare', label: 'Server files prepared' },
command: { state: 'attempted', message: 'Komodo deployment creation attempted.' } });
const spec = buildSpec({ game, name, port, hostDir, plutoKey, recipe });
const createRes = await komodoCreateDeployment(spec);
const createRes = await observeCommand(operation, () => komodoCreateDeployment(spec));
if (!createRes.ok) {
recordProgress(operation, { command: { state: createRes.kind === 'unreachable' || (createRes.httpStatus !== undefined && createRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: createRes.message } });
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
return { ok: false, kind: 'komodo', message: `CreateDeployment: ${createRes.message}` };
}
recordProgress(operation, { command: { state: 'accepted', message: 'Komodo created the deployment.' },
checkpoint: { key: 'remote-deployment-created', phase: 'create-deployment', label: 'Komodo deployment created' } });
// Step 3: DB row — insert *before* execute/Deploy so the sweep sees the
// server immediately even if Deploy hangs.
@@ -446,23 +471,31 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
void applyDeploymentTags(name, game, mapEntry.canonical);
// Step 5: execute/Deploy — actually start the container.
deploymentStage(operation, 'Starting container');
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operation);
const deployRes = await komodoDeploy(name);
db().transaction(() => {
deploymentStage(operation, 'Starting container', { phase: 'container-action',
checkpoint: { key: 'create-deployment', label: 'Deployment created and server registered' },
command: { state: 'attempted', message: 'Container startup submission attempted.' } });
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operation);
})();
const deployRes = await observeCommand(operation, () => komodoDeploy(name));
if (!deployRes.ok) {
if (deployRes.kind === 'unreachable' || (deployRes.httpStatus !== undefined && deployRes.httpStatus >= 500)) {
// A timeout or server error may arrive after the remote boot began.
// Preserve files, identity, and capacity until readiness is known.
recordDeploymentEvent(operation, 'Startup response uncertain; verifying the existing server: ' + deployRes.message);
recordProgress(operation, { command: { state: 'uncertain', message: 'Startup may have begun: ' + deployRes.message } });
return { ok: true, name, port, deploymentId: operation };
}
// Container never started — undo everything so retrying is clean.
recordProgress(operation, { command: { state: 'rejected', message: 'Startup request rejected: ' + deployRes.message } });
await komodoDeleteDeployment(name).catch(() => {});
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
await rm(hostDir, { recursive: true, force: true }).catch(() => {});
return { ok: false, kind: 'komodo', message: `Deploy: ${deployRes.message}` };
}
recordProgress(operation, { command: { state: 'accepted', message: 'Komodo accepted startup; readiness still requires verification.' },
checkpoint: { key: 'container-action', label: 'Container startup accepted' } });
return { ok: true, name, port, deploymentId: operation };
}
@@ -515,7 +548,8 @@ export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerRe
const operation = db().transaction(() => {
const id = beginDeployment(name, 'destroy', triggeredBy);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting to verify the server is empty');
deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
blocker: { code: 'eligibility', message: 'Waiting for safe removal eligibility checks.', automatic: true } });
return id;
})();
return { ok: true, warnings: [], deploymentId: operation };
@@ -528,6 +562,13 @@ export function processDestructions(): Promise<void> {
return destructionTick;
}
function betaRemovalEvidence(server: string, message: string): ProvisioningBlocker {
const beta = db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') ORDER BY b.id LIMIT 1`)
.get(server) as { id: number } | undefined;
return { code: 'active-beta', message, automatic: true, href: beta ? `/betas/${beta.id}` : null, label: 'Blocking beta' };
}
async function runDestructions(): Promise<void> {
const queued = db().prepare(`SELECT id, server, stage, server_deleted_at FROM server_deployments WHERE action = 'destroy' AND state = 'waiting'
AND (server_deleted_at IS NULL OR stage = ?) ORDER BY started_at`).all(REMOVING_SERVER) as { id: string; server: string; stage: string; server_deleted_at: string | null }[];
@@ -553,37 +594,53 @@ async function runDestructions(): Promise<void> {
}
if (!getServer(operation.server)) {
if (hasActiveCreation(operation.server)) {
deploymentStage(operation.id, 'Waiting for server creation to finish');
const creatingOperation = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health') LIMIT 1`).get(operation.server) as { id: string } | undefined;
deploymentStage(operation.id, 'Waiting for server creation to finish', { phase: 'eligibility', checks: [],
blocker: { code: 'creation-pending', message: 'Waiting for server creation to finish', automatic: true,
href: creatingOperation ? `/provisioning/${encodeURIComponent(creatingOperation.id)}` : null, label: 'Creation operation' } });
continue;
}
finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' });
continue;
}
const blocker = activeBetaRemovalBlocker(operation.server);
if (blocker) { deploymentStage(operation.id, blocker); continue; }
if (blocker) { deploymentStage(operation.id, blocker, { phase: 'eligibility', checks: [], blocker: betaRemovalEvidence(operation.server, blocker) }); continue; }
// A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
let players = await probePlayerCount(operation.server);
const evidence = await probePlayerEvidence(operation.server);
let players = evidence.players;
let occupancy: NonNullable<ProgressUpdate['checks']>[number] = { key: 'occupancy', label: 'Player occupancy',
result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting', source: evidence.source,
checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players };
if (players === null) {
const health = await komodoInspectHealthy(operation.server);
if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) {
players = 0;
occupancy = { key: 'stopped', label: 'Stopped container permits removal', result: 'pass', source: 'komodo', actual: health.state ?? null };
}
}
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
// Beta ownership may have changed while the network probes were in flight.
const latestBlocker = activeBetaRemovalBlocker(operation.server);
if (latestBlocker) { deploymentStage(operation.id, latestBlocker); continue; }
if (latestBlocker) { deploymentStage(operation.id, latestBlocker, { phase: 'eligibility', checks: [occupancy], blocker: betaRemovalEvidence(operation.server, latestBlocker) }); continue; }
if (players !== 0) {
deploymentStage(operation.id, players === null
const message = players === null
? 'Waiting for player count: server could not be reached'
: 'Waiting for server to empty: ' + players + ' player(s) online');
: 'Waiting for server to empty: ' + players + ' player(s) online';
deploymentStage(operation.id, message, { phase: 'eligibility', checks: [occupancy],
blocker: { code: players === null ? 'players-unknown' : 'players-present', message, automatic: true } });
continue;
}
if (!claimRemoval(operation.id)) continue;
deploymentStage(operation.id, 'Server is empty (0 players verified)');
deploymentStage(operation.id, 'Server is empty (0 players verified)', { phase: 'eligibility', checks: [occupancy], blocker: null,
checkpoint: { key: 'eligibility', label: 'Removal eligibility verified' } });
const result = await destroyServerUnlocked(operation.server, operation.id);
finishDeployment(operation.id, result);
} catch (err) {
const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string };
if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message);
if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message, {
phase: 'eligibility', checks: [], blocker: { code: 'players-unknown', message: (err as Error).message, automatic: true }
});
else if (row.state !== 'cancelled') finishDeployment(operation.id, { ok: false, message: (err as Error).message });
} finally { destroying.delete(operation.server); }
}
@@ -601,11 +658,15 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
// 1: stop + remove the container. execute/DestroyDeployment queues the
// teardown; the response comes back before the container is actually gone.
deploymentStage(operation, 'Stopping and removing container');
const destroyRes = await komodoDestroyDeployment(name);
deploymentStage(operation, 'Stopping and removing container', { phase: 'remove-container', blocker: null, checks: [],
checkpoint: { key: 'eligibility', label: 'Removal eligibility verified' },
command: { state: 'attempted', message: 'Container removal submission attempted.' } });
const destroyRes = await observeCommand(operation, () => komodoDestroyDeployment(name));
if (!destroyRes.ok) {
recordProgress(operation, { command: { state: destroyRes.kind === 'unreachable' || (destroyRes.httpStatus !== undefined && destroyRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: destroyRes.message } });
return { ok: false, kind: 'komodo', message: 'DestroyDeployment failed: ' + destroyRes.message, warnings };
}
recordProgress(operation, { command: { state: 'accepted', message: 'Komodo accepted teardown; removal is not yet confirmed.' } });
// 2: wait for the container to actually disappear. Komodo refuses to
// DeleteDeployment while a container still exists for it, so poll the
@@ -619,12 +680,19 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
}
// 3: remove the deployment entity itself.
deploymentStage(operation, 'Deleting Komodo deployment');
const deleteRes = await komodoDeleteDeployment(name);
deploymentStage(operation, 'Deleting Komodo deployment', { phase: 'delete-deployment',
command: { state: 'attempted', message: 'Komodo deployment deletion attempted.' } });
const deleteRes = await observeCommand(operation, () => komodoDeleteDeployment(name));
if (!deleteRes.ok) {
recordProgress(operation, { command: { state: deleteRes.kind === 'unreachable' || (deleteRes.httpStatus !== undefined && deleteRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: deleteRes.message } });
return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings };
}
deploymentStage(operation, REMOVING_SERVER, { phase: 'cleanup', blocker: null, checks: [],
command: { state: 'accepted', message: 'Komodo confirmed deployment deletion.' },
checkpoints: [{ key: 'remove-container', label: 'Container removal confirmed by deployment deletion' },
{ key: 'delete-deployment', label: 'Komodo deployment deletion confirmed' }] });
return cleanupDestroyedServer(name, operation);
}
@@ -635,12 +703,14 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
const warnings: string[] = [];
// Persist the checkpoint before local work so failed cleanup can resume without RCON.
// Legacy operations already recorded this same stage after Komodo deletion.
deploymentStage(operation, REMOVING_SERVER);
deploymentStage(operation, REMOVING_SERVER, { phase: 'cleanup', blocker: null, checks: [],
checkpoint: { key: 'delete-deployment', label: 'Komodo deployment deletion confirmed' } });
// Host directory. Fall back to derived path if host_dir wasn't set on
// legacy rows.
const hostDir = row?.host_dir ?? serverDir(name);
try {
await rm(hostDir, { recursive: true, force: true });
recordProgress(operation, { checkpoint: { key: 'files-removed', label: 'Instance files removed', phase: 'cleanup' } });
} catch (err) {
warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
}
@@ -648,7 +718,14 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
// The registry trigger preserves history and marks its old server identity.
// Live inventory, rollback snapshots, and restart slots still cascade away.
try {
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
db().transaction(() => {
const recovery = db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ? AND healthy_at IS NULL').get(name) as { operation_id: string | null } | undefined;
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
if (recovery?.operation_id) recordProgress(recovery.operation_id, { recovery: { state: 'released', operationId: recovery.operation_id,
message: 'Recovery capacity released when the old instance registry was removed.' } });
recordProgress(operation, { checkpoint: { key: 'registry-removed', label: 'Server registration removed', phase: 'cleanup' } });
if (!warnings.length) recordProgress(operation, { checkpoint: { key: 'cleanup', label: 'Local files and registry cleanup completed' } });
})();
} catch (err) {
return {
ok: false,
@@ -684,8 +761,11 @@ async function provisionCreation(id: string): Promise<CreateServerResult> {
{ ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
if (!result.ok) finishDeployment(id, result);
else {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for Komodo health');
db().transaction(() => {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', checks: [],
blocker: { code: 'container-readiness', message: 'Waiting for container readiness.', automatic: true } });
})();
}
return result;
} catch (err) {
@@ -693,9 +773,13 @@ async function provisionCreation(id: string): Promise<CreateServerResult> {
if (server?.game_port === job.port) {
// A lost response may have happened after Komodo started the server.
// Keep its slot and observe readiness instead of starting another boot.
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message);
deploymentStage(id, 'Verifying existing server after interrupted startup');
db().transaction(() => {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message);
deploymentStage(id, 'Verifying existing server after interrupted startup', { phase: 'verify-container', checks: [],
command: { state: 'uncertain', message: 'Startup response interrupted: ' + (err as Error).message },
blocker: { code: 'container-readiness', message: 'Observing the existing instance after an uncertain startup.', automatic: true } });
})();
} else finishDeployment(id, { ok: false, message: (err as Error).message });
throw err;
} finally { creating = false; }
@@ -722,19 +806,32 @@ export function creationQueue() {
return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
}
function creationBlocker(id: string): string | null {
const dependency = db().prepare(`SELECT d.server, d.state FROM server_creation_dependencies dep
function creationBlocker(id: string): ProvisioningBlocker | null {
const dependency = db().prepare(`SELECT d.id, d.server, d.state FROM server_creation_dependencies dep
JOIN server_deployments d ON d.id = dep.removal_id
WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { server: string; state: string } | undefined;
if (dependency) return `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`;
WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { id: string; server: string; state: string } | undefined;
if (dependency) return { code: 'removal-dependency', message: `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`,
automatic: true, href: `/provisioning/${encodeURIComponent(dependency.id)}`, label: 'Required removal' };
const occupied = db().prepare(`SELECT s.name FROM servers s JOIN server_creation_jobs j ON s.game_port = j.port
WHERE j.deployment_id = ? LIMIT 1`).get(id) as { name: string } | undefined;
return occupied ? `Waiting for port to be released by ${occupied.name}` : null;
return occupied ? { code: 'port-occupied', message: `Waiting for port to be released by ${occupied.name}`, automatic: true,
href: `/servers/${encodeURIComponent(occupied.name)}`, label: 'Server using this port' } : null;
}
function stillCheckingCreation(id: string): boolean {
return !!db().prepare(`SELECT 1 FROM server_deployments d JOIN server_creation_jobs j ON j.deployment_id = d.id
JOIN servers s ON s.name = j.server AND s.game_port = j.port
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state = 'awaiting-health'`).get(id);
}
async function runCreations(): Promise<CreationOutcomes> {
const outcomes: CreationOutcomes = new Map();
if (creating || destroying.size > 0) return outcomes;
if (creating || destroying.size > 0) {
for (const job of creationQueue().queued) recordProgress(job.id, { phase: 'startup-queue', blocker: {
code: 'startup-worker-busy', message: 'Waiting for in-flight instance creation or removal to finish.', automatic: true
} });
return outcomes;
}
// Restart recovery is observation-only: never repeat external creation or
// deployment. A known interrupted startup must retain its readiness gate.
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
@@ -752,7 +849,8 @@ async function runCreations(): Promise<CreationOutcomes> {
AND newer.rowid > d.rowid AND newer.action = 'create')`).get(id);
if (removing) {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for requested server removal after interrupted startup');
deploymentStage(id, 'Waiting for requested server removal after interrupted startup', { checks: [],
blocker: { code: 'removal-pending', message: 'Waiting for the requested removal; the interrupted startup still occupies capacity.', automatic: true } });
}
}
@@ -766,21 +864,51 @@ async function runCreations(): Promise<CreationOutcomes> {
finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' });
continue;
}
if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
const removal = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server) as { id: string } | undefined;
if (removal) {
recordProgress(id, { checks: [], blocker: { code: 'removal-pending', message: 'Creation verification is held while server removal is pending.', automatic: true,
href: `/provisioning/${encodeURIComponent(removal.id)}`, label: 'Removal operation' } });
continue;
}
const checks: NonNullable<ProgressUpdate['checks']> = [];
try {
const komodo = await komodoInspectHealthy(job.server);
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
if (!stillCheckingCreation(id)) continue;
checks.push({ key: 'container', label: 'Container readiness', source: 'komodo', result: komodo.ok ? 'pass' : 'waiting',
checkedAt: new Date().toISOString(), actual: komodo.state ?? null, message: komodo.ok ? null : komodo.reason });
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason, { phase: 'verify-container', checks,
blocker: { code: 'container-readiness', message: komodo.reason, automatic: true } }); continue; }
const status = await probeServerStatus(job.server, job.submitted_at);
if (!status.ok) { deploymentStage(id, 'Waiting for RCON or fresh telemetry health: ' + status.reason); continue; }
if (!stillCheckingCreation(id)) continue;
checks.push({ key: 'game', label: 'Game status', source: status.ok ? status.source : null, result: status.ok ? 'pass' : 'waiting',
checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null, message: status.ok ? null : status.reason });
if (!status.ok) { deploymentStage(id, 'Waiting for RCON or fresh telemetry health: ' + status.reason, { phase: 'verify-game', checks,
blocker: { code: 'game-readiness', message: status.reason, automatic: true } }); continue; }
const actualMap = status.map;
checks.push({ key: 'map', label: 'Selected engine map', source: status.source, result: actualMap === job.expected_map ? 'pass' : 'waiting',
checkedAt: status.observedAt, receivedAt: status.receivedAt, expected: job.expected_map, actual: actualMap });
if (actualMap !== job.expected_map) {
deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', ' + (status.source === 'rcon' ? 'RCON' : 'telemetry') + ' reports ' + (actualMap ?? 'no map'));
const message = 'Waiting for selected map: expected ' + job.expected_map + ', ' + (status.source === 'rcon' ? 'RCON' : 'telemetry') + ' reports ' + (actualMap ?? 'no map');
deploymentStage(id, message, { phase: 'verify-map', checks, blocker: { code: 'map-mismatch', message, automatic: true,
href: `/servers/${encodeURIComponent(job.server)}?tab=config`, label: 'Map configuration' } });
continue;
}
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap);
finishDeployment(id, { ok: true });
db().transaction(() => {
deploymentStage(id, 'Verified Komodo healthy, ' + (status.source === 'rcon' ? 'RCON healthy' : 'fresh telemetry') + ', and map ' + actualMap, {
phase: 'verify-map', checks, blocker: null, checkpoints: [
{ key: 'container-action', label: 'Container startup verified' },
{ key: 'verify-container', label: 'Container readiness verified' },
{ key: 'verify-game', label: 'Game status verified' },
{ key: 'verify-map', label: 'Selected engine map verified' }
]
});
finishDeployment(id, { ok: true });
})();
} catch (err) {
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message);
if (!stillCheckingCreation(id)) continue;
checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', source: null, result: 'unknown', message: (err as Error).message });
deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message, { phase: checks.length > 1 ? 'verify-game' : 'verify-container', checks,
blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true } });
}
}
@@ -790,7 +918,9 @@ async function runCreations(): Promise<CreationOutcomes> {
const available = Math.max(0, fleetSettings().createConcurrency - queue.active.length);
const eligible = queue.queued.filter(({ id }) => {
const blocker = creationBlocker(id);
if (blocker) deploymentStage(id, blocker);
if (blocker) deploymentStage(id, blocker.message, { phase: 'startup-queue', blocker });
else recordProgress(id, { phase: 'startup-queue', blocker: { code: available ? 'startup-queue' : 'startup-capacity',
message: available ? 'Eligible for startup admission in request order.' : 'Waiting for startup capacity to be released.', automatic: true } });
return !blocker;
});
for (const { id } of eligible.slice(0, available)) {
@@ -801,7 +931,7 @@ async function runCreations(): Promise<CreationOutcomes> {
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
const blocker = creationBlocker(id);
if (blocker) {
deploymentStage(id, blocker);
deploymentStage(id, blocker.message, { phase: 'startup-queue', blocker });
continue;
}
try { outcomes.set(id, await provisionCreation(id)); }