From c2ad5cc95a3aee8bd9bbd12a2d69453b53857818 Mon Sep 17 00:00:00 2001 From: josh Date: Thu, 17 Sep 2026 17:14:36 -0400 Subject: [PATCH] Queue server creation and verify health and selected map before completion --- README.md | 2 +- src/lib/server/db/schema.sql | 9 ++ src/lib/server/fleet/create.ts | 134 ++++++++++++++---- src/lib/server/fleet/deployments.ts | 4 +- src/lib/server/integrations/komodo.ts | 8 +- src/lib/server/pipeline/worker.ts | 4 +- .../ui/components/DeployServerDialog.svelte | 9 +- src/routes/api/servers/+server.ts | 10 +- src/routes/deployments/+page.svelte | 10 +- src/routes/deployments/[id]/+page.svelte | 14 +- tests/unit/create-server.test.ts | 66 ++++++++- tests/unit/komodo.test.ts | 12 +- 12 files changed, 232 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 74c9759..a5d22db 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,6 @@ RCON parsing (fake UDP socket). Campaigns (`/campaigns`) contains script/config rollouts, drift reconciliation, restart progress, file diffs, and rollbacks. Existing rollout records are preserved in the original database tables. The campaign API lives at `/api/campaigns`; legacy rollout detail API endpoints remain available for compatibility. -Deployments (`/deployments`, `/api/deployments`) now tracks server creations and destructions with live stages and persistent history, including failures, warnings, and interrupted operations. History starts with operations performed after this update and survives server removal. Creation completion means the server was provisioned and Komodo accepted the container start request; runtime health remains on the Fleet page. +Deployments (`/deployments`, `/api/deployments`) now tracks server creations and destructions with live stages and persistent history, including failures, warnings, and interrupted operations. History starts with operations performed after this update and survives server removal. New creation requests return HTTP 202 with a deployment ID and open the deployment detail page. Creation is complete only when Komodo reports healthy, RCON responds successfully, and the RCON map exactly matches the selected map. Queued requests reserve their server name and port; queued creation and pending health checks survive application restarts. The detail page records startup progress and explains any health or map mismatch that is still blocking completion. Each server deployment opens a detail page with its flow and timestamped activity. Destroy uses an in-app confirmation modal and immediately returns a queued deployment (`DELETE /api/servers/:name` returns HTTP 202 with `deploymentId`). The worker waits for a fresh RCON player count of zero before teardown; occupied or unreachable servers remain queued and are checked each worker tick. Waiting requests survive application restarts, while interrupted teardown is marked for inspection. Completed destruction stays on the deployment detail page. diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index 2f81a97..5bc302e 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -276,3 +276,12 @@ CREATE TABLE IF NOT EXISTS server_deployment_events ( message TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_server_deployment_events ON server_deployment_events(deployment_id, id); + +CREATE TABLE IF NOT EXISTS server_creation_jobs ( + deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE, + server TEXT NOT NULL, + port INTEGER NOT NULL, + game_mode TEXT NOT NULL, + map_slug TEXT NOT NULL, + expected_map TEXT NOT NULL +); diff --git a/src/lib/server/fleet/create.ts b/src/lib/server/fleet/create.ts index dc77de3..c6b9cb6 100644 --- a/src/lib/server/fleet/create.ts +++ b/src/lib/server/fleet/create.ts @@ -1,4 +1,4 @@ -import { rconGetPlayerCount } from '../integrations/rcon.js'; +import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js'; import { beginDeployment, deploymentStage, finishDeployment } from './deployments.js'; import { mkdir, chmod, rm, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; @@ -6,7 +6,7 @@ import { config, destOfKind } from '../config.js'; import { db } from '../db/index.js'; import { getServerKey, saveServerKey } from './server-keys.js'; import { getServer } from './servers.js'; -import { t5ZombieMapBySlug, type GameMode } from './map-aliases.js'; +import { t5ZombieMapBySlug, type GameMode, type T5MapEntry } from './map-aliases.js'; import { komodoCreateDeployment, komodoDeploy, @@ -40,7 +40,7 @@ export type CreateServerFailure = | 'db'; // shouldn't happen, but wraps INSERT failures export type CreateServerResult = - | { ok: true; name: string; port: number } + | { ok: true; name: string; port: number; deploymentId: string } | { ok: false; kind: CreateServerFailure; message: string }; export type DestroyServerResult = @@ -62,11 +62,14 @@ export interface CreateServerInput { */ function nextLobbyNumber(baseName: string): number | null { const rows = db() - .prepare('SELECT name FROM servers WHERE name LIKE ?') + .prepare(`SELECT name FROM servers WHERE name LIKE ? UNION + SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id + WHERE d.state IN ('queued', 'running', 'awaiting-health')`) .all(`${baseName}_%`) as Array<{ name: string }>; const taken = new Set(); const prefix = `${baseName}_`; for (const r of rows) { + if (!r.name.startsWith(prefix)) continue; const tail = r.name.slice(prefix.length); if (/^\d+$/.test(tail)) taken.add(Number.parseInt(tail, 10)); } @@ -83,7 +86,9 @@ function nextLobbyNumber(baseName: string): number | null { */ function nextAvailablePort(): number | null { const rows = db() - .prepare('SELECT game_port FROM servers WHERE game_port IS NOT NULL') + .prepare(`SELECT game_port FROM servers WHERE game_port IS NOT NULL UNION + SELECT j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id + WHERE d.state IN ('queued', 'running', 'awaiting-health')`) .all() as Array<{ game_port: number }>; const taken = new Set(rows.map((r) => r.game_port)); for (let p = config.portRange.min; p <= config.portRange.max; p++) { @@ -139,25 +144,32 @@ function buildSpec(args: { } let creating = false; -export async function createServer(input: CreateServerInput): Promise { - if (creating || destroying.size > 0) return { ok: false, kind: 'no-slot', message: 'Another server operation is in progress. Please retry shortly.' }; - creating = true; - let operation: string | undefined; - try { - const result = await createServerUnlocked(input, (name) => { - operation = beginDeployment(name, 'create', input.createdBy); - return operation; - }); - if (operation) finishDeployment(operation, result); - return result; - } catch (err) { - if (operation) finishDeployment(operation, { ok: false, message: (err as Error).message }); - throw err; - } - finally { creating = false; } +interface PreparedCreation { + ok: true; name: string; port: number; plutoKey: string; mapEntry: T5MapEntry; } -async function createServerUnlocked(input: CreateServerInput, begin: (name: string) => string): Promise { +export function queueCreateServer(input: CreateServerInput): CreateServerResult { + return db().transaction((): CreateServerResult => { + const prepared = prepareCreation(input); + if (!prepared.ok) return prepared; + const id = beginDeployment(prepared.name, 'create', input.createdBy); + db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id); + db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)') + .run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical); + deploymentStage(id, 'Queued for server creation'); + return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id }; + })(); +} + +// Await provisioning for internal callers; completion still requires worker health verification. +export async function createServer(input: CreateServerInput): Promise { + if (creating || destroying.size > 0) return { ok: false, kind: 'no-slot', message: 'Another server operation is in progress. Please retry shortly.' }; + const queued = queueCreateServer(input); + if (!queued.ok) return queued; + return await provisionCreation(queued.deploymentId); +} + +function prepareCreation(input: CreateServerInput): PreparedCreation | Extract { if (input.gameMode !== 'zombie') { return { ok: false, kind: 'unknown-map', message: `unsupported game mode: ${input.gameMode}` }; } @@ -198,7 +210,11 @@ async function createServerUnlocked(input: CreateServerInput, begin: (name: stri try { saveServerKey(name, plutoKey); } catch (err) { return { ok: false, kind: 'bad-request', message: (err as Error).message }; } - const operation = begin(name); + return { ok: true, name, port, plutoKey, mapEntry }; +} + +async function createServerUnlocked(input: CreateServerInput, prepared: PreparedCreation, operation: string): Promise { + const { name, port, plutoKey, mapEntry } = prepared; deploymentStage(operation, 'Preparing server files'); const hostDir = `${config.serversRoot}/${name}`; const initialConfig = getPromotedServerConfig('t5', mapEntry.canonical); @@ -304,7 +320,7 @@ async function createServerUnlocked(input: CreateServerInput, begin: (name: stri // now instead of on the next 15s tick. void kick().catch((err) => console.error('[createServer] post-create kick failed:', err)); - return { ok: true, name, port }; + return { ok: true, name, port, deploymentId: operation }; } /** @@ -453,5 +469,75 @@ async function destroyServerUnlocked(name: string, operation: string): Promise { + const job = db().prepare('SELECT j.*, d.triggered_by FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE j.deployment_id = ?').get(id) as CreationJob; + creating = true; + db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(id); + try { + const mapEntry = t5ZombieMapBySlug(job.map_slug); + const plutoKey = getServerKey(job.server); + if (!mapEntry || !plutoKey) { + const result = { ok: false as const, kind: 'bad-request' as const, message: 'Saved server key or selected map is no longer available' }; + finishDeployment(id, result); + return result; + } + const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by }, + { ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id); + if (!result.ok) finishDeployment(id, result); + else { + db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id); + deploymentStage(id, 'Waiting for Komodo health'); + } + return result; + } catch (err) { + finishDeployment(id, { ok: false, message: (err as Error).message }); + throw err; + } finally { creating = false; } +} + +let creationTick: Promise | null = null; +export function processCreations(): Promise { + if (creationTick) return creationTick; + creationTick = runCreations().finally(() => { creationTick = null; }); + return creationTick; +} + +async function runCreations(): Promise { + if (creating || destroying.size > 0) return; + const queued = db().prepare("SELECT id FROM server_deployments WHERE action = 'create' AND state = 'queued' ORDER BY started_at").all() as { id: string }[]; + for (const { id } of queued) { + if (creating || destroying.size > 0) break; + await provisionCreation(id).catch(err => console.error('[creation] provisioning failed:', err)); + } + const checking = db().prepare("SELECT j.* FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state = 'awaiting-health'").all() as CreationJob[]; + for (const job of checking) { + const id = job.deployment_id; + if (!getServer(job.server)) { + finishDeployment(id, { ok: false, message: 'Server was removed before creation health checks completed' }); + continue; + } + if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue; + try { + const komodo = await komodoInspectHealthy(job.server); + if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; } + const rcon = await rconStatusResult(job.server); + if (!rcon.ok) { deploymentStage(id, 'Waiting for RCON health: ' + rcon.reason); continue; } + const actualMap = rcon.status.map; + if (actualMap !== job.expected_map) { + deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', RCON reports ' + (actualMap ?? 'no map')); + continue; + } + deploymentStage(id, 'Verified Komodo healthy, RCON healthy, and map ' + actualMap); + finishDeployment(id, { ok: true }); + } catch (err) { + deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message); + } + } } diff --git a/src/lib/server/fleet/deployments.ts b/src/lib/server/fleet/deployments.ts index d39a070..41bc0e8 100644 --- a/src/lib/server/fleet/deployments.ts +++ b/src/lib/server/fleet/deployments.ts @@ -21,8 +21,8 @@ export function recordDeploymentEvent(id: string, message: string): void { } export function getDeployment(id: string) { - const deployment = db().prepare(`SELECT d.*, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists - FROM server_deployments d WHERE d.id = ?`).get(id); + const deployment = db().prepare(`SELECT d.*, j.expected_map, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists + FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id); if (!deployment) return null; const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id); return { deployment, events }; diff --git a/src/lib/server/integrations/komodo.ts b/src/lib/server/integrations/komodo.ts index 055e7ab..d888545 100644 --- a/src/lib/server/integrations/komodo.ts +++ b/src/lib/server/integrations/komodo.ts @@ -277,7 +277,7 @@ const STATE_KEYS = new Set([ 'docker_status' ]); -function extractContainerState(payload: unknown): string | null { +export function extractContainerState(payload: unknown): string | null { const seen = new Set(); let found: string | null = null; @@ -297,6 +297,12 @@ function extractContainerState(payload: unknown): string | null { const status = so['Status']; const running = so['Running']; const health = so['Health']; + // A running process can still be starting or unhealthy. Honor an + // explicit Docker health check before accepting its running state. + if ((running === true || status === 'running') && health && typeof health === 'object') { + const hs = (health as Record)['Status']; + if (typeof hs === 'string' && hs.toLowerCase() !== 'healthy') { found = hs; return; } + } if (typeof status === 'string') { found = status; return; } if (running === true) { found = 'running'; return; } if (health && typeof health === 'object') { diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index 2f7682d..2ba9847 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -1,4 +1,4 @@ -import { processDestructions } from '../fleet/create.js'; +import { processDestructions, processCreations } from '../fleet/create.js'; import { sweep } from './sweep.js'; import { betaTick } from './beta-runner.js'; import { emit } from './bus.js'; @@ -28,6 +28,8 @@ function tick(): Promise { } async function runTick(): Promise { + try { await processCreations(); } + catch (err) { console.error('[worker] creation error:', err); } try { await processDestructions(); } catch (err) { console.error('[worker] destruction error:', err); } try { diff --git a/src/lib/ui/components/DeployServerDialog.svelte b/src/lib/ui/components/DeployServerDialog.svelte index 453827f..1c88b98 100644 --- a/src/lib/ui/components/DeployServerDialog.svelte +++ b/src/lib/ui/components/DeployServerDialog.svelte @@ -66,7 +66,8 @@ const [keysRes, serversRes] = await Promise.all([fetch('/api/server-keys'), fetch('/api/servers')]); if (!keysRes.ok || !serversRes.ok) throw new Error('Could not load saved keys and server slots. Reopen this dialog to retry.'); savedIds = (await keysRes.json()).keys.map((k: { server_id: string }) => k.server_id); - deployedIds = (await serversRes.json()).servers.map((s: { name: string }) => s.name); + const fleet = await serversRes.json(); + deployedIds = [...fleet.servers.map((s: { name: string }) => s.name), ...(fleet.reservedNames ?? [])]; keysLoaded = true; } catch (err) { flash = (err as Error).message; } } @@ -100,15 +101,15 @@ headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ gameMode, mapSlug: selectedMap.slug, expectedName: previewName, plutoKey: plutoKey.trim() || undefined }) }); - const body = (await res.json()) as { ok: boolean; name?: string; port?: number; error?: string; kind?: string }; - if (!body.ok || !body.name) { + const body = (await res.json()) as { ok: boolean; name?: string; port?: number; deploymentId?: string; error?: string; kind?: string }; + if (!res.ok || !body.ok || !body.deploymentId) { flash = body.error ?? 'Deploy failed'; if (body.kind === 'no-slot') await loadKeys(); deploying = false; return; } close(); - await goto(`/servers/${encodeURIComponent(body.name)}`); + await goto(`/deployments/${encodeURIComponent(body.deploymentId)}`); } catch (err) { flash = (err as Error).message; deploying = false; diff --git a/src/routes/api/servers/+server.ts b/src/routes/api/servers/+server.ts index 4f199f5..050db4a 100644 --- a/src/routes/api/servers/+server.ts +++ b/src/routes/api/servers/+server.ts @@ -1,7 +1,9 @@ +import { db } from '$lib/server/db/index.js'; +import { kick } from '$lib/server/pipeline/worker.js'; import { json } from '@sveltejs/kit'; import { listServers, heldByBeta } from '$lib/server/fleet/servers.js'; import { classifyHealth } from '$lib/server/fleet/health.js'; -import { createServer } from '$lib/server/fleet/create.js'; +import { queueCreateServer } from '$lib/server/fleet/create.js'; import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js'; import type { RconPlayer } from '$lib/server/integrations/rcon.js'; import type { GameMode } from '$lib/server/fleet/map-aliases.js'; @@ -29,6 +31,7 @@ export async function GET() { })); return json({ servers, + reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server), integrations: { rcon: rconConfigured(), komodo: komodoConfigured(), @@ -51,7 +54,7 @@ export async function POST({ request }: { request: Request }) { { status: 400 } ); } - const result = await createServer({ + const result = queueCreateServer({ gameMode: body.gameMode as GameMode, mapSlug: body.mapSlug, plutoKey: body.plutoKey, @@ -64,5 +67,6 @@ export async function POST({ request }: { request: Request }) { : 500; return json({ ok: false, error: result.message, kind: result.kind }, { status }); } - return json({ ok: true, name: result.name, port: result.port }); + void kick().catch((err) => console.error('[create] worker kick failed:', err)); + return json({ ok: true, name: result.name, port: result.port, deploymentId: result.deploymentId }, { status: 202 }); } diff --git a/src/routes/deployments/+page.svelte b/src/routes/deployments/+page.svelte index 16eebf3..8573b70 100644 --- a/src/routes/deployments/+page.svelte +++ b/src/routes/deployments/+page.svelte @@ -8,7 +8,7 @@ interface Deployment { id: string; server: string; action: 'create' | 'destroy'; - state: 'waiting' | 'running' | 'done' | 'failed' | 'warning' | 'interrupted'; + state: 'queued' | 'awaiting-health' | 'waiting' | 'running' | 'done' | 'failed' | 'warning' | 'interrupted'; stage: string; started_at: string; finished_at: string | null; triggered_by: string; error: string | null; warnings: string; server_exists: number; } @@ -19,8 +19,8 @@ let action = $state('all'); let search = $state(''); let refreshing = false; - let active = $derived(deployments.filter(d => (d.state === 'running' || d.state === 'waiting'))); - let history = $derived(deployments.filter(d => d.state !== 'running' && d.state !== 'waiting' && + let active = $derived(deployments.filter(d => ['queued', 'awaiting-health', 'running', 'waiting'].includes(d.state))); + let history = $derived(deployments.filter(d => !['queued', 'awaiting-health', 'running', 'waiting'].includes(d.state) && (action === 'all' || d.action === action) && d.server.toLowerCase().includes(search.toLowerCase()))); async function load() { if (refreshing) return; @@ -41,7 +41,7 @@ function fmt(value: string) { return new Date(value).toLocaleString(); } function warnings(d: Deployment): string[] { try { return JSON.parse(d.warnings); } catch { return []; } } function status(d: Deployment) { - return { waiting: 'Waiting for empty', running: 'In flight', done: 'Completed', failed: 'Failed', warning: 'Completed with warnings', interrupted: 'Interrupted' }[d.state]; + return { queued: 'Queued', 'awaiting-health': 'Verifying health', waiting: 'Waiting for empty', running: 'In flight', done: 'Completed', failed: 'Failed', warning: 'Completed with warnings', interrupted: 'Interrupted' }[d.state]; } @@ -78,7 +78,7 @@ {status(d)} -

{d.state === 'done' ? (d.action === 'create' ? 'Server created; container start requested.' : 'Server destroyed.') : d.stage}

+

{d.state === 'done' ? (d.action === 'create' ? 'Server creation completed.' : 'Server destroyed.') : d.stage}

Started {fmt(d.started_at)}{#if d.finished_at}Finished {fmt(d.finished_at)}{/if}By {d.triggered_by}
{#if d.error}

{d.error}

{/if} {#each warnings(d) as warning}

{warning}

{/each} diff --git a/src/routes/deployments/[id]/+page.svelte b/src/routes/deployments/[id]/+page.svelte index e7e2965..5e4e959 100644 --- a/src/routes/deployments/[id]/+page.svelte +++ b/src/routes/deployments/[id]/+page.svelte @@ -6,7 +6,7 @@ import Badge from '$ui/components/primitives/Badge.svelte'; interface Deployment { - id: string; server: string; action: 'create' | 'destroy'; state: string; + id: string; expected_map?: string; server: string; action: 'create' | 'destroy'; state: string; stage: string; started_at: string; finished_at: string | null; triggered_by: string; error: string | null; warnings: string; server_exists: number; } @@ -15,24 +15,25 @@ let error = $state(''); let refreshing = false; let deployment = $derived(detail?.deployment); - let active = $derived(deployment?.state === 'waiting' || deployment?.state === 'running'); + let active = $derived(['queued', 'awaiting-health', 'waiting', 'running'].includes(deployment?.state ?? '')); let warnings = $derived.by(() => { try { return JSON.parse(deployment?.warnings ?? '[]') as string[]; } catch { return []; } }); let steps = $derived(deployment?.action === 'destroy' ? ['Request destruction', 'Wait for empty server', 'Stop and remove container', 'Delete Komodo deployment', 'Remove server files and registry'] - : ['Request creation', 'Prepare server files', 'Create Komodo deployment', 'Start container']); + : ['Request creation', 'Prepare server files', 'Create Komodo deployment', 'Start container', 'Verify Komodo health', 'Verify RCON health', 'Verify selected map']); let currentStep = $derived.by(() => { if (!deployment) return 0; const stages = deployment.action === 'destroy' ? ['Starting', 'Waiting', 'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry'] - : ['Starting', 'Preparing server files', 'Creating Komodo deployment', 'Starting container']; + : ['Starting', 'Preparing server files', 'Creating Komodo deployment', 'Starting container', 'Waiting for Komodo health', 'Waiting for RCON health', 'Waiting for selected map']; if (deployment.state === 'waiting') return 1; + if (deployment.state === 'awaiting-health' && deployment.stage.startsWith('Waiting for health verification')) return 4; if (deployment.state === 'done') return stages.length; return Math.max(0, stages.findIndex(stage => deployment.stage.startsWith(stage))); }); function stateLabel(state: string) { - return ({ waiting: 'Waiting for empty', running: 'In flight', done: 'Completed', failed: 'Failed', warning: 'Completed with warnings', interrupted: 'Interrupted' } as Record)[state] ?? state; + return ({ queued: 'Queued', 'awaiting-health': 'Verifying health', waiting: 'Waiting for empty', running: 'In flight', done: 'Completed', failed: 'Failed', warning: 'Completed with warnings', interrupted: 'Interrupted' } as Record)[state] ?? state; } async function load() { if (refreshing) return; @@ -65,10 +66,11 @@
{stateLabel(deployment.state)} {#if deployment.server_exists}View server{/if}
-

{deployment.state === 'done' ? (deployment.action === 'destroy' ? 'Server destroyed' : 'Server created; container start requested') : deployment.stage}

+

{deployment.state === 'done' ? (deployment.action === 'destroy' ? 'Server destroyed' : 'Server creation completed') : deployment.stage}

{#if deployment.state === 'waiting'}

The server stays online while players finish. Player count is checked automatically; destruction starts only after a live check reports zero players.

{/if} {#if deployment.error}{/if} {#each warnings as warning}

{warning}

{/each} + {#if deployment.expected_map}

Expected map: {deployment.expected_map}

{/if}
Started
{fmt(deployment.started_at)}
Finished
{deployment.finished_at ? fmt(deployment.finished_at) : 'In progress'}
Requested by
{deployment.triggered_by}
diff --git a/tests/unit/create-server.test.ts b/tests/unit/create-server.test.ts index f3bc9d4..bbf40ec 100644 --- a/tests/unit/create-server.test.ts +++ b/tests/unit/create-server.test.ts @@ -13,7 +13,11 @@ vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })), komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true })) })); -vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(async () => 0) })); +vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ + rconGetPlayerCount: vi.fn(async () => 0), + rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'localhost', port: 27016 }, rawReply: '', + status: { players: 0, map: 'zombie_cosmodrome', gametype: null, roster: [] } })) +})); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); let fixture: Fixture; @@ -54,6 +58,62 @@ async function promotedConfig() { } describe('server creation map selection', () => { + it('returns a queued creation before provisioning and reserves its name and port', async () => { + const { POST } = await import('../../src/routes/api/servers/+server.js'); + const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); + const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { closeDb } = await import('../../src/lib/server/db/index.js'); + const response = await POST({ request: new Request('http://localhost/api/servers', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(input) + }) }); + expect(response.status).toBe(202); + const body = await response.json(); + expect(body).toMatchObject({ name: serverName, deploymentId: expect.any(String) }); + expect(komodoCreateDeployment).not.toHaveBeenCalled(); + expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'queued', expected_map: 'zombie_cosmodrome' }); + expect(queueCreateServer({ ...input, expectedName: serverName })).toMatchObject({ ok: false, kind: 'no-slot' }); + const second = queueCreateServer(input); + expect(second).toMatchObject({ ok: true, name: 't5_zombie_cosmodrome_2' }); + if (second.ok) expect(second.port).not.toBe(body.port); + closeDb(); + interruptDeployments(); + expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'queued' }); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }).mockResolvedValueOnce({ ok: false, reason: 'starting' }); + await Promise.all([processCreations(), processCreations()]); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); + expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null }); + }); + + it('completes creation only when Komodo, RCON, and the selected map all pass', async () => { + const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); + const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); + const created = await createServer(input); + if (!created.ok) throw new Error('fixture creation failed'); + const id = created.deploymentId; + expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null }); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }); + await processCreations(); + expect(rconStatusResult).not.toHaveBeenCalled(); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); + vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'not ready' }); + await processCreations(); + expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', stage: 'Waiting for RCON health: timeout' }); + for (const map of [null, 'zombie_moon']) { + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); + vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'localhost', port: created.port }, rawReply: '', status: { players: 0, map, gametype: null, roster: [] } }); + await processCreations(); + expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null, stage: expect.stringContaining('expected zombie_cosmodrome') }); + } + interruptDeployments(); + expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health' }); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); + await processCreations(); + expect(getDeployment(id)?.deployment).toMatchObject({ state: 'done', finished_at: expect.any(String), stage: 'Verified Komodo healthy, RCON healthy, and map zombie_cosmodrome' }); + }); + it('returns a deployment URL identifier immediately and serves its lifecycle detail', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js'); @@ -116,7 +176,7 @@ describe('server creation map selection', () => { }); it('records in-flight creation and retains both operations after destruction', async () => { - const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); + const { createServer, destroyServer, processDestructions, processCreations } = await import('../../src/lib/server/fleet/create.js'); const { listDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { komodoDeploy, komodoInspectHealthy, komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoDeploy).mockImplementationOnce(async () => { @@ -124,6 +184,8 @@ describe('server creation map selection', () => { return { ok: true }; }); expect((await createServer(input)).ok).toBe(true); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); + await processCreations(); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }); vi.mocked(komodoDestroyDeployment).mockImplementationOnce(async () => { expect(listDeployments()).toContainEqual(expect.objectContaining({ action: 'destroy', state: 'running' })); diff --git a/tests/unit/komodo.test.ts b/tests/unit/komodo.test.ts index 9118352..7422003 100644 --- a/tests/unit/komodo.test.ts +++ b/tests/unit/komodo.test.ts @@ -1,5 +1,15 @@ import { describe, expect, it } from 'vitest'; -import { extractUdpPort } from '../../src/lib/server/integrations/komodo.js'; +import { extractUdpPort, extractContainerState } from '../../src/lib/server/integrations/komodo.js'; + +describe('container readiness', () => { + it.each(['starting', 'unhealthy'])('does not accept a running container whose health is %s', (health) => { + expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: health } } })).toBe(health); + }); + it('accepts healthy running containers and preserves stopped states', () => { + expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: 'healthy' } } })).toBe('running'); + expect(extractContainerState({ State: { Status: 'exited', Running: false, Health: { Status: 'healthy' } } })).toBe('exited'); + }); +}); describe('extractUdpPort', () => { it('returns the published port from docker-compose-style object entries', () => {