josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add provisioning lifecycle actions

Browse Source 
Rename Deployments to Provisioning with compatible redirects and APIs. Add durable Start, Restart, Stop, and Rebuild operations alongside Create and Destroy; rebuild wipes the instance and recreates it in the same slot and port.

Add lifecycle, migration, recovery, and Komodo command tests. Type checks and production build pass; the full suite retains a telemetry assertion failure reproduced on the unchanged branch.

Refs #39
This commit is contained in:
josh
2026-09-27 14:20:04 -04:00
parent 0b0e5b5bdb
commit cfed584614
42 changed files with 886 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/create.ts
+77 -10
View File
@@ -42,7 +42,7 @@ export type CreateServerResult =
export type DestroyServerResult =
| { ok: true; warnings: string[] }
| { ok: false; kind: 'not-found' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] };
| { ok: false; kind: 'not-found' | 'conflict' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] };
export interface CreateServerInput {
game?: ZombieGame; // omitted by older clients: T5
@@ -63,7 +63,7 @@ export function nextLobbyNumber(baseName: string, additionalNames: string[] = []
const rows = db()
.prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
.all(`${baseName}_%`) as Array<{ name: string }>;
const taken = new Set<number>();
const prefix = `${baseName}_`;
@@ -87,7 +87,7 @@ export function nextAvailablePort(additionalPorts: number[] = [], releasingServe
const rows = db()
.prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
.all() as Array<{ name: string; game_port: number }>;
const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
for (let p = config.portRange.min; p <= config.portRange.max; p++) {
@@ -229,6 +229,63 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
})();
}
/** Reserve the same identity and port before removing any part of the instance. */
export function queueRebuildServer(name: string): CreateServerResult {
const server = getServer(name);
const game = gameOfServer(name);
const map = server?.map_slug && zombieMapBySlug(game, server.map_slug);
if (!server || !map || !server.game_port || server.komodo_type !== 'deployment' || server.komodo_name !== name
|| (server.host_dir && server.host_dir !== serverDir(name))) {
return { ok: false, kind: 'bad-request', message: 'Rebuild requires a managed server with its original map, directory, and port.' };
}
if (!config.komodo.targetServer || !config.rcon.password) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' };
if (!getServerKey(name)) return { ok: false, kind: 'bad-request', message: `Save a server key for ${name} in Keys before rebuilding.` };
// Validate that the replacement setup can be composed before destruction.
const recipe = captureRecipe(game, map.slug);
return db().transaction(() => {
const id = beginDeployment(name, 'rebuild');
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
.run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe));
deploymentStage(id, 'Waiting to rebuild the empty server');
return { ok: true as const, name, port: server.game_port!, deploymentId: id };
})();
}
/** Caller has just verified emptiness and excluded conflicting work. */
export async function rebuildServer(id: string): Promise<void> {
const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.id = ? AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
if (!job) return;
if (creating || destroying.size || creationQueue().active.length >= config.createConcurrency) {
// The cleanup checkpoint must survive capacity waits and app restarts.
if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding');
return;
}
// Recheck prerequisites while the original instance still exists.
if (!getServerKey(job.server)) { finishDeployment(id, { ok: false, message: 'Saved server key was removed; rebuild did not begin.' }); return; }
try { captureRecipe(gameOfServer(job.server), job.map_slug); }
catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); return; }
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
destroying.add(job.server);
try {
const result = job.stage === REMOVING_SERVER
? await cleanupDestroyedServer(job.server, id) : await destroyServerUnlocked(job.server, id);
if (!result.ok || result.warnings.length) {
finishDeployment(id, { ok: false, message: result.ok ? 'Old server files could not be completely removed.' : result.message, warnings: result.warnings });
return;
}
db().transaction(() => {
db().prepare('UPDATE server_rebuild_jobs SET removed_at = ? WHERE deployment_id = ?').run(new Date().toISOString(), id);
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
deploymentStage(id, 'Queued for server creation');
})();
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
finally { destroying.delete(job.server); }
}
function nextName(input: CreateServerInput): string | null {
const base = `${input.game ?? 't5'}_${input.gameMode}_${input.mapSlug}`;
const lobby = nextLobbyNumber(base);
@@ -437,6 +494,10 @@ function hasActiveCreation(name: string): boolean {
}
export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action IN ('start','restart','stop','rebuild')
AND state IN ('waiting','queued','running')`).get(name)) {
return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] };
}
const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
@@ -485,7 +546,11 @@ async function runDestructions(): Promise<void> {
continue;
}
// A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
const players = await rconGetPlayerCount(operation.server);
let players = await rconGetPlayerCount(operation.server);
if (players === null) {
const health = await komodoInspectHealthy(operation.server);
if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
}
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
if (players !== 0) {
deploymentStage(operation.id, players === null
@@ -580,10 +645,12 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
};
}
const unfinished = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' AND state = 'awaiting-health'").all(name) as { id: string }[];
for (const { id } of unfinished) finishDeployment(id, { ok: false, message: 'Server destroyed before creation health checks completed' });
const unfinished = db().prepare("SELECT id, action FROM server_deployments WHERE server = ? AND state = 'awaiting-health'").all(name) as { id: string; action: string }[];
for (const { id, action } of unfinished) finishDeployment(id, { ok: false, message: action === 'create'
? 'Server destroyed before creation health checks completed' : 'Server destroyed before operation verification completed' });
return { ok: true, warnings };
}␍interface CreationJob {
}
interface CreationJob {
deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
}
@@ -636,7 +703,7 @@ export async function processCreations(): Promise<void> {
export function creationQueue() {
type Entry = { id: string; server: string; state: string; stage: string };
const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments
WHERE action = 'create' AND state IN ('queued', 'running', 'awaiting-health')
WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
ORDER BY started_at, rowid`).all() as Entry[];
return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
}
@@ -656,7 +723,7 @@ async function runCreations(): Promise<CreationOutcomes> {
if (creating || destroying.size > 0) return outcomes;
// Restart recovery is observation-only: never repeat external creation or
// deployment. A known interrupted startup must retain its readiness gate.
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action = 'create' AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
for (const { id } of interrupted) {
if (resumeVerification(id)) continue;
// A queued removal does not make the interrupted container disappear.
@@ -717,7 +784,7 @@ async function runCreations(): Promise<CreationOutcomes> {
// API calls can cancel a selected job or resume an older startup while
// a previous provision awaits Komodo. Recheck without expanding this pass.
if (creationQueue().active.length >= config.createConcurrency) break;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action = 'create' AND state = 'queued'").get(id)) continue;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
const blocker = creationBlocker(id);
if (blocker) {
deploymentStage(id, blocker);