Queue server creation and verify health and selected map before completion

src/lib/server/db/schema.sql

@@ -276,3 +276,12 @@ CREATE TABLE IF NOT EXISTS server_deployment_events (
     message TEXT NOT NULL
 );
 CREATE INDEX IF NOT EXISTS idx_server_deployment_events ON server_deployment_events(deployment_id, id);
+
+CREATE TABLE IF NOT EXISTS server_creation_jobs (
+    deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE,
+    server TEXT NOT NULL,
+    port INTEGER NOT NULL,
+    game_mode TEXT NOT NULL,
+    map_slug TEXT NOT NULL,
+    expected_map TEXT NOT NULL
+);

src/lib/server/fleet/create.ts

@@ -1,4 +1,4 @@
-import { rconGetPlayerCount } from '../integrations/rcon.js';
+import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
 import { beginDeployment, deploymentStage, finishDeployment } from './deployments.js';
 import { mkdir, chmod, rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
@@ -6,7 +6,7 @@ import { config, destOfKind } from '../config.js';
 import { db } from '../db/index.js';
 import { getServerKey, saveServerKey } from './server-keys.js';
 import { getServer } from './servers.js';
-import { t5ZombieMapBySlug, type GameMode } from './map-aliases.js';
+import { t5ZombieMapBySlug, type GameMode, type T5MapEntry } from './map-aliases.js';
 import {
     komodoCreateDeployment,
     komodoDeploy,
@@ -40,7 +40,7 @@ export type CreateServerFailure =
     | 'db';             // shouldn't happen, but wraps INSERT failures
 
 export type CreateServerResult =
-    | { ok: true; name: string; port: number }
+    | { ok: true; name: string; port: number; deploymentId: string }
     | { ok: false; kind: CreateServerFailure; message: string };
 
 export type DestroyServerResult =
@@ -62,11 +62,14 @@ export interface CreateServerInput {
  */
 function nextLobbyNumber(baseName: string): number | null {
     const rows = db()
-        .prepare('SELECT name FROM servers WHERE name LIKE ?')
+        .prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
+            SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
+            WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
         .all(`${baseName}_%`) as Array<{ name: string }>;
     const taken = new Set<number>();
     const prefix = `${baseName}_`;
     for (const r of rows) {
+        if (!r.name.startsWith(prefix)) continue;
         const tail = r.name.slice(prefix.length);
         if (/^\d+$/.test(tail)) taken.add(Number.parseInt(tail, 10));
     }
@@ -83,7 +86,9 @@ function nextLobbyNumber(baseName: string): number | null {
  */
 function nextAvailablePort(): number | null {
     const rows = db()
-        .prepare('SELECT game_port FROM servers WHERE game_port IS NOT NULL')
+        .prepare(`SELECT game_port FROM servers WHERE game_port IS NOT NULL UNION
+            SELECT j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
+            WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
         .all() as Array<{ game_port: number }>;
     const taken = new Set(rows.map((r) => r.game_port));
     for (let p = config.portRange.min; p <= config.portRange.max; p++) {
@@ -139,25 +144,32 @@ function buildSpec(args: {
 }
 
 let creating = false;
-export async function createServer(input: CreateServerInput): Promise<CreateServerResult> {
-    if (creating || destroying.size > 0) return { ok: false, kind: 'no-slot', message: 'Another server operation is in progress. Please retry shortly.' };
-    creating = true;
-    let operation: string | undefined;
-    try {
-        const result = await createServerUnlocked(input, (name) => {
-            operation = beginDeployment(name, 'create', input.createdBy);
-            return operation;
-        });
-        if (operation) finishDeployment(operation, result);
-        return result;
-    } catch (err) {
-        if (operation) finishDeployment(operation, { ok: false, message: (err as Error).message });
-        throw err;
-    }
-    finally { creating = false; }
+interface PreparedCreation {
+    ok: true; name: string; port: number; plutoKey: string; mapEntry: T5MapEntry;
 }
 
-async function createServerUnlocked(input: CreateServerInput, begin: (name: string) => string): Promise<CreateServerResult> {
+export function queueCreateServer(input: CreateServerInput): CreateServerResult {
+    return db().transaction((): CreateServerResult => {
+        const prepared = prepareCreation(input);
+        if (!prepared.ok) return prepared;
+        const id = beginDeployment(prepared.name, 'create', input.createdBy);
+        db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
+        db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
+            .run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical);
+        deploymentStage(id, 'Queued for server creation');
+        return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id };
+    })();
+}
+
+// Await provisioning for internal callers; completion still requires worker health verification.
+export async function createServer(input: CreateServerInput): Promise<CreateServerResult> {
+    if (creating || destroying.size > 0) return { ok: false, kind: 'no-slot', message: 'Another server operation is in progress. Please retry shortly.' };
+    const queued = queueCreateServer(input);
+    if (!queued.ok) return queued;
+    return await provisionCreation(queued.deploymentId);
+}
+
+function prepareCreation(input: CreateServerInput): PreparedCreation | Extract<CreateServerResult, { ok: false }> {
     if (input.gameMode !== 'zombie') {
         return { ok: false, kind: 'unknown-map', message: `unsupported game mode: ${input.gameMode}` };
     }
@@ -198,7 +210,11 @@ async function createServerUnlocked(input: CreateServerInput, begin: (name: stri
     try { saveServerKey(name, plutoKey); }
     catch (err) { return { ok: false, kind: 'bad-request', message: (err as Error).message }; }
 
-    const operation = begin(name);
+    return { ok: true, name, port, plutoKey, mapEntry };
+}
+
+async function createServerUnlocked(input: CreateServerInput, prepared: PreparedCreation, operation: string): Promise<CreateServerResult> {
+    const { name, port, plutoKey, mapEntry } = prepared;
     deploymentStage(operation, 'Preparing server files');
     const hostDir = `${config.serversRoot}/${name}`;
     const initialConfig = getPromotedServerConfig('t5', mapEntry.canonical);
@@ -304,7 +320,7 @@ async function createServerUnlocked(input: CreateServerInput, begin: (name: stri
     // now instead of on the next 15s tick.
     void kick().catch((err) => console.error('[createServer] post-create kick failed:', err));
 
-    return { ok: true, name, port };
+    return { ok: true, name, port, deploymentId: operation };
 }
 
 /**
@@ -453,5 +469,75 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
         };
     }
 
+    const unfinished = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' AND state = 'awaiting-health'").all(name) as { id: string }[];
+    for (const { id } of unfinished) finishDeployment(id, { ok: false, message: 'Server destroyed before creation health checks completed' });
     return { ok: true, warnings };
+}␍interface CreationJob {
+    deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
+}
+
+async function provisionCreation(id: string): Promise<CreateServerResult> {
+    const job = db().prepare('SELECT j.*, d.triggered_by FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE j.deployment_id = ?').get(id) as CreationJob;
+    creating = true;
+    db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(id);
+    try {
+        const mapEntry = t5ZombieMapBySlug(job.map_slug);
+        const plutoKey = getServerKey(job.server);
+        if (!mapEntry || !plutoKey) {
+            const result = { ok: false as const, kind: 'bad-request' as const, message: 'Saved server key or selected map is no longer available' };
+            finishDeployment(id, result);
+            return result;
+        }
+        const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by },
+            { ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
+        if (!result.ok) finishDeployment(id, result);
+        else {
+            db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
+            deploymentStage(id, 'Waiting for Komodo health');
+        }
+        return result;
+    } catch (err) {
+        finishDeployment(id, { ok: false, message: (err as Error).message });
+        throw err;
+    } finally { creating = false; }
+}
+
+let creationTick: Promise<void> | null = null;
+export function processCreations(): Promise<void> {
+    if (creationTick) return creationTick;
+    creationTick = runCreations().finally(() => { creationTick = null; });
+    return creationTick;
+}
+
+async function runCreations(): Promise<void> {
+    if (creating || destroying.size > 0) return;
+    const queued = db().prepare("SELECT id FROM server_deployments WHERE action = 'create' AND state = 'queued' ORDER BY started_at").all() as { id: string }[];
+    for (const { id } of queued) {
+        if (creating || destroying.size > 0) break;
+        await provisionCreation(id).catch(err => console.error('[creation] provisioning failed:', err));
+    }
+    const checking = db().prepare("SELECT j.* FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state = 'awaiting-health'").all() as CreationJob[];
+    for (const job of checking) {
+        const id = job.deployment_id;
+        if (!getServer(job.server)) {
+            finishDeployment(id, { ok: false, message: 'Server was removed before creation health checks completed' });
+            continue;
+        }
+        if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
+        try {
+            const komodo = await komodoInspectHealthy(job.server);
+            if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
+            const rcon = await rconStatusResult(job.server);
+            if (!rcon.ok) { deploymentStage(id, 'Waiting for RCON health: ' + rcon.reason); continue; }
+            const actualMap = rcon.status.map;
+            if (actualMap !== job.expected_map) {
+                deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', RCON reports ' + (actualMap ?? 'no map'));
+                continue;
+            }
+            deploymentStage(id, 'Verified Komodo healthy, RCON healthy, and map ' + actualMap);
+            finishDeployment(id, { ok: true });
+        } catch (err) {
+            deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message);
+        }
+    }
 }

src/lib/server/fleet/deployments.ts

@@ -21,8 +21,8 @@ export function recordDeploymentEvent(id: string, message: string): void {
 }
 
 export function getDeployment(id: string) {
-    const deployment = db().prepare(`SELECT d.*, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
-        FROM server_deployments d WHERE d.id = ?`).get(id);
+    const deployment = db().prepare(`SELECT d.*, j.expected_map, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
+        FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id);
     if (!deployment) return null;
     const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id);
     return { deployment, events };

src/lib/server/integrations/komodo.ts

@@ -277,7 +277,7 @@ const STATE_KEYS = new Set([
     'docker_status'
 ]);
 
-function extractContainerState(payload: unknown): string | null {
+export function extractContainerState(payload: unknown): string | null {
     const seen = new Set<unknown>();
     let found: string | null = null;
 
@@ -297,6 +297,12 @@ function extractContainerState(payload: unknown): string | null {
             const status = so['Status'];
             const running = so['Running'];
             const health = so['Health'];
+            // A running process can still be starting or unhealthy. Honor an
+            // explicit Docker health check before accepting its running state.
+            if ((running === true || status === 'running') && health && typeof health === 'object') {
+                const hs = (health as Record<string, unknown>)['Status'];
+                if (typeof hs === 'string' && hs.toLowerCase() !== 'healthy') { found = hs; return; }
+            }
             if (typeof status === 'string') { found = status; return; }
             if (running === true) { found = 'running'; return; }
             if (health && typeof health === 'object') {

src/lib/server/pipeline/worker.ts

@@ -1,4 +1,4 @@
-import { processDestructions } from '../fleet/create.js';
+import { processDestructions, processCreations } from '../fleet/create.js';
 import { sweep } from './sweep.js';
 import { betaTick } from './beta-runner.js';
 import { emit } from './bus.js';
@@ -28,6 +28,8 @@ function tick(): Promise<void> {
 }
 
 async function runTick(): Promise<void> {
+    try { await processCreations(); }
+    catch (err) { console.error('[worker] creation error:', err); }
     try { await processDestructions(); }
     catch (err) { console.error('[worker] destruction error:', err); }
     try {

src/lib/ui/components/DeployServerDialog.svelte

@@ -66,7 +66,8 @@
             const [keysRes, serversRes] = await Promise.all([fetch('/api/server-keys'), fetch('/api/servers')]);
             if (!keysRes.ok || !serversRes.ok) throw new Error('Could not load saved keys and server slots. Reopen this dialog to retry.');
             savedIds = (await keysRes.json()).keys.map((k: { server_id: string }) => k.server_id);
-            deployedIds = (await serversRes.json()).servers.map((s: { name: string }) => s.name);
+            const fleet = await serversRes.json();
+            deployedIds = [...fleet.servers.map((s: { name: string }) => s.name), ...(fleet.reservedNames ?? [])];
             keysLoaded = true;
         } catch (err) { flash = (err as Error).message; }
     }
@@ -100,15 +101,15 @@
                 headers: { 'Content-Type': 'application/json' },
                 body: JSON.stringify({ gameMode, mapSlug: selectedMap.slug, expectedName: previewName, plutoKey: plutoKey.trim() || undefined })
             });
-            const body = (await res.json()) as { ok: boolean; name?: string; port?: number; error?: string; kind?: string };
-            if (!body.ok || !body.name) {
+            const body = (await res.json()) as { ok: boolean; name?: string; port?: number; deploymentId?: string; error?: string; kind?: string };
+            if (!res.ok || !body.ok || !body.deploymentId) {
                 flash = body.error ?? 'Deploy failed';
                 if (body.kind === 'no-slot') await loadKeys();
                 deploying = false;
                 return;
             }
             close();
-            await goto(`/servers/${encodeURIComponent(body.name)}`);
+            await goto(`/deployments/${encodeURIComponent(body.deploymentId)}`);
         } catch (err) {
             flash = (err as Error).message;
             deploying = false;

src/routes/api/servers/+server.ts

@@ -1,7 +1,9 @@
+import { db } from '$lib/server/db/index.js';
+import { kick } from '$lib/server/pipeline/worker.js';
 import { json } from '@sveltejs/kit';
 import { listServers, heldByBeta } from '$lib/server/fleet/servers.js';
 import { classifyHealth } from '$lib/server/fleet/health.js';
-import { createServer } from '$lib/server/fleet/create.js';
+import { queueCreateServer } from '$lib/server/fleet/create.js';
 import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js';
 import type { RconPlayer } from '$lib/server/integrations/rcon.js';
 import type { GameMode } from '$lib/server/fleet/map-aliases.js';
@@ -29,6 +31,7 @@ export async function GET() {
     }));
     return json({
         servers,
+        reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
         integrations: {
             rcon: rconConfigured(),
             komodo: komodoConfigured(),
@@ -51,7 +54,7 @@ export async function POST({ request }: { request: Request }) {
             { status: 400 }
         );
     }
-    const result = await createServer({
+    const result = queueCreateServer({
         gameMode: body.gameMode as GameMode,
         mapSlug: body.mapSlug,
         plutoKey: body.plutoKey,
@@ -64,5 +67,6 @@ export async function POST({ request }: { request: Request }) {
             : 500;
         return json({ ok: false, error: result.message, kind: result.kind }, { status });
     }
-    return json({ ok: true, name: result.name, port: result.port });
+    void kick().catch((err) => console.error('[create] worker kick failed:', err));
+    return json({ ok: true, name: result.name, port: result.port, deploymentId: result.deploymentId }, { status: 202 });
 }

src/routes/deployments/+page.svelte

@@ -8,7 +8,7 @@
 
     interface Deployment {
         id: string; server: string; action: 'create' | 'destroy';
-        state: 'waiting' | 'running' | 'done' | 'failed' | 'warning' | 'interrupted';
+        state: 'queued' | 'awaiting-health' | 'waiting' | 'running' | 'done' | 'failed' | 'warning' | 'interrupted';
         stage: string; started_at: string; finished_at: string | null;
         triggered_by: string; error: string | null; warnings: string; server_exists: number;
     }
@@ -19,8 +19,8 @@
     let action = $state('all');
     let search = $state('');
     let refreshing = false;
-    let active = $derived(deployments.filter(d => (d.state === 'running' || d.state === 'waiting')));
-    let history = $derived(deployments.filter(d => d.state !== 'running' && d.state !== 'waiting' &&
+    let active = $derived(deployments.filter(d => ['queued', 'awaiting-health', 'running', 'waiting'].includes(d.state)));
+    let history = $derived(deployments.filter(d => !['queued', 'awaiting-health', 'running', 'waiting'].includes(d.state) &&
         (action === 'all' || d.action === action) && d.server.toLowerCase().includes(search.toLowerCase())));
     async function load() {
         if (refreshing) return;
@@ -41,7 +41,7 @@
     function fmt(value: string) { return new Date(value).toLocaleString(); }
     function warnings(d: Deployment): string[] { try { return JSON.parse(d.warnings); } catch { return []; } }
     function status(d: Deployment) {
-        return { waiting: 'Waiting for empty', running: 'In flight', done: 'Completed', failed: 'Failed', warning: 'Completed with warnings', interrupted: 'Interrupted' }[d.state];
+        return { queued: 'Queued', 'awaiting-health': 'Verifying health', waiting: 'Waiting for empty', running: 'In flight', done: 'Completed', failed: 'Failed', warning: 'Completed with warnings', interrupted: 'Interrupted' }[d.state];
     }
 </script>
 
@@ -78,7 +78,7 @@
             </div>
             <Badge variant={d.state === 'done' ? 'online' : d.state === 'failed' ? 'danger' : 'warning'}>{status(d)}</Badge>
         </div>
-        <p>{d.state === 'done' ? (d.action === 'create' ? 'Server created; container start requested.' : 'Server destroyed.') : d.stage}</p>
+        <p>{d.state === 'done' ? (d.action === 'create' ? 'Server creation completed.' : 'Server destroyed.') : d.stage}</p>
         <div class="meta"><span>Started {fmt(d.started_at)}</span>{#if d.finished_at}<span>Finished {fmt(d.finished_at)}</span>{/if}<span>By {d.triggered_by}</span></div>
         {#if d.error}<p class="error">{d.error}</p>{/if}
         {#each warnings(d) as warning}<p class="error">{warning}</p>{/each}

src/routes/deployments/[id]/+page.svelte

@@ -6,7 +6,7 @@
     import Badge from '$ui/components/primitives/Badge.svelte';
 
     interface Deployment {
-        id: string; server: string; action: 'create' | 'destroy'; state: string;
+        id: string; expected_map?: string; server: string; action: 'create' | 'destroy'; state: string;
         stage: string; started_at: string; finished_at: string | null;
         triggered_by: string; error: string | null; warnings: string; server_exists: number;
     }
@@ -15,24 +15,25 @@
     let error = $state('');
     let refreshing = false;
     let deployment = $derived(detail?.deployment);
-    let active = $derived(deployment?.state === 'waiting' || deployment?.state === 'running');
+    let active = $derived(['queued', 'awaiting-health', 'waiting', 'running'].includes(deployment?.state ?? ''));
     let warnings = $derived.by(() => {
         try { return JSON.parse(deployment?.warnings ?? '[]') as string[]; } catch { return []; }
     });
     let steps = $derived(deployment?.action === 'destroy'
         ? ['Request destruction', 'Wait for empty server', 'Stop and remove container', 'Delete Komodo deployment', 'Remove server files and registry']
-        : ['Request creation', 'Prepare server files', 'Create Komodo deployment', 'Start container']);
+        : ['Request creation', 'Prepare server files', 'Create Komodo deployment', 'Start container', 'Verify Komodo health', 'Verify RCON health', 'Verify selected map']);
     let currentStep = $derived.by(() => {
         if (!deployment) return 0;
         const stages = deployment.action === 'destroy'
             ? ['Starting', 'Waiting', 'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry']
-            : ['Starting', 'Preparing server files', 'Creating Komodo deployment', 'Starting container'];
+            : ['Starting', 'Preparing server files', 'Creating Komodo deployment', 'Starting container', 'Waiting for Komodo health', 'Waiting for RCON health', 'Waiting for selected map'];
         if (deployment.state === 'waiting') return 1;
+        if (deployment.state === 'awaiting-health' && deployment.stage.startsWith('Waiting for health verification')) return 4;
         if (deployment.state === 'done') return stages.length;
         return Math.max(0, stages.findIndex(stage => deployment.stage.startsWith(stage)));
     });
     function stateLabel(state: string) {
-        return ({ waiting: 'Waiting for empty', running: 'In flight', done: 'Completed', failed: 'Failed', warning: 'Completed with warnings', interrupted: 'Interrupted' } as Record<string, string>)[state] ?? state;
+        return ({ queued: 'Queued', 'awaiting-health': 'Verifying health', waiting: 'Waiting for empty', running: 'In flight', done: 'Completed', failed: 'Failed', warning: 'Completed with warnings', interrupted: 'Interrupted' } as Record<string, string>)[state] ?? state;
     }
     async function load() {
         if (refreshing) return;
@@ -65,10 +66,11 @@
             <div class="top"><Badge variant={deployment.state === 'done' ? 'online' : deployment.state === 'failed' ? 'danger' : 'warning'}>{stateLabel(deployment.state)}</Badge>
                 {#if deployment.server_exists}<a href="/servers/{encodeURIComponent(deployment.server)}">View server</a>{/if}
             </div>
-            <h2>{deployment.state === 'done' ? (deployment.action === 'destroy' ? 'Server destroyed' : 'Server created; container start requested') : deployment.stage}</h2>
+            <h2>{deployment.state === 'done' ? (deployment.action === 'destroy' ? 'Server destroyed' : 'Server creation completed') : deployment.stage}</h2>
             {#if deployment.state === 'waiting'}<p>The server stays online while players finish. Player count is checked automatically; destruction starts only after a live check reports zero players.</p>{/if}
             {#if deployment.error}<p role="alert" class="error">{deployment.error}</p>{/if}
             {#each warnings as warning}<p class="warning">{warning}</p>{/each}
+            {#if deployment.expected_map}<p>Expected map: <strong>{deployment.expected_map}</strong></p>{/if}
             <dl><div><dt>Started</dt><dd>{fmt(deployment.started_at)}</dd></div><div><dt>Finished</dt><dd>{deployment.finished_at ? fmt(deployment.finished_at) : 'In progress'}</dd></div><div><dt>Requested by</dt><dd>{deployment.triggered_by}</dd></div></dl>
         </section>
         <section>