Queue server creation and verify health and selected map before completion

src/lib/server/db/schema.sql

@@ -276,3 +276,12 @@ CREATE TABLE IF NOT EXISTS server_deployment_events (
     message TEXT NOT NULL
 );
 CREATE INDEX IF NOT EXISTS idx_server_deployment_events ON server_deployment_events(deployment_id, id);
+
+CREATE TABLE IF NOT EXISTS server_creation_jobs (
+    deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE,
+    server TEXT NOT NULL,
+    port INTEGER NOT NULL,
+    game_mode TEXT NOT NULL,
+    map_slug TEXT NOT NULL,
+    expected_map TEXT NOT NULL
+);

src/lib/server/fleet/create.ts

@@ -1,4 +1,4 @@
-import { rconGetPlayerCount } from '../integrations/rcon.js';
+import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
 import { beginDeployment, deploymentStage, finishDeployment } from './deployments.js';
 import { mkdir, chmod, rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
@@ -6,7 +6,7 @@ import { config, destOfKind } from '../config.js';
 import { db } from '../db/index.js';
 import { getServerKey, saveServerKey } from './server-keys.js';
 import { getServer } from './servers.js';
-import { t5ZombieMapBySlug, type GameMode } from './map-aliases.js';
+import { t5ZombieMapBySlug, type GameMode, type T5MapEntry } from './map-aliases.js';
 import {
     komodoCreateDeployment,
     komodoDeploy,
@@ -40,7 +40,7 @@ export type CreateServerFailure =
     | 'db';             // shouldn't happen, but wraps INSERT failures
 
 export type CreateServerResult =
-    | { ok: true; name: string; port: number }
+    | { ok: true; name: string; port: number; deploymentId: string }
     | { ok: false; kind: CreateServerFailure; message: string };
 
 export type DestroyServerResult =
@@ -62,11 +62,14 @@ export interface CreateServerInput {
  */
 function nextLobbyNumber(baseName: string): number | null {
     const rows = db()
-        .prepare('SELECT name FROM servers WHERE name LIKE ?')
+        .prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
+            SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
+            WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
         .all(`${baseName}_%`) as Array<{ name: string }>;
     const taken = new Set<number>();
     const prefix = `${baseName}_`;
     for (const r of rows) {
+        if (!r.name.startsWith(prefix)) continue;
         const tail = r.name.slice(prefix.length);
         if (/^\d+$/.test(tail)) taken.add(Number.parseInt(tail, 10));
     }
@@ -83,7 +86,9 @@ function nextLobbyNumber(baseName: string): number | null {
  */
 function nextAvailablePort(): number | null {
     const rows = db()
-        .prepare('SELECT game_port FROM servers WHERE game_port IS NOT NULL')
+        .prepare(`SELECT game_port FROM servers WHERE game_port IS NOT NULL UNION
+            SELECT j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
+            WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
         .all() as Array<{ game_port: number }>;
     const taken = new Set(rows.map((r) => r.game_port));
     for (let p = config.portRange.min; p <= config.portRange.max; p++) {
@@ -139,25 +144,32 @@ function buildSpec(args: {
 }
 
 let creating = false;
-export async function createServer(input: CreateServerInput): Promise<CreateServerResult> {
-    if (creating || destroying.size > 0) return { ok: false, kind: 'no-slot', message: 'Another server operation is in progress. Please retry shortly.' };
-    creating = true;
-    let operation: string | undefined;
-    try {
-        const result = await createServerUnlocked(input, (name) => {
-            operation = beginDeployment(name, 'create', input.createdBy);
-            return operation;
-        });
-        if (operation) finishDeployment(operation, result);
-        return result;
-    } catch (err) {
-        if (operation) finishDeployment(operation, { ok: false, message: (err as Error).message });
-        throw err;
-    }
-    finally { creating = false; }
+interface PreparedCreation {
+    ok: true; name: string; port: number; plutoKey: string; mapEntry: T5MapEntry;
 }
 
-async function createServerUnlocked(input: CreateServerInput, begin: (name: string) => string): Promise<CreateServerResult> {
+export function queueCreateServer(input: CreateServerInput): CreateServerResult {
+    return db().transaction((): CreateServerResult => {
+        const prepared = prepareCreation(input);
+        if (!prepared.ok) return prepared;
+        const id = beginDeployment(prepared.name, 'create', input.createdBy);
+        db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
+        db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
+            .run(id, prepared.name, prepared.port, input.gameMode, prepared.mapEntry.slug, prepared.mapEntry.canonical);
+        deploymentStage(id, 'Queued for server creation');
+        return { ok: true, name: prepared.name, port: prepared.port, deploymentId: id };
+    })();
+}
+
+// Await provisioning for internal callers; completion still requires worker health verification.
+export async function createServer(input: CreateServerInput): Promise<CreateServerResult> {
+    if (creating || destroying.size > 0) return { ok: false, kind: 'no-slot', message: 'Another server operation is in progress. Please retry shortly.' };
+    const queued = queueCreateServer(input);
+    if (!queued.ok) return queued;
+    return await provisionCreation(queued.deploymentId);
+}
+
+function prepareCreation(input: CreateServerInput): PreparedCreation | Extract<CreateServerResult, { ok: false }> {
     if (input.gameMode !== 'zombie') {
         return { ok: false, kind: 'unknown-map', message: `unsupported game mode: ${input.gameMode}` };
     }
@@ -198,7 +210,11 @@ async function createServerUnlocked(input: CreateServerInput, begin: (name: stri
     try { saveServerKey(name, plutoKey); }
     catch (err) { return { ok: false, kind: 'bad-request', message: (err as Error).message }; }
 
-    const operation = begin(name);
+    return { ok: true, name, port, plutoKey, mapEntry };
+}
+
+async function createServerUnlocked(input: CreateServerInput, prepared: PreparedCreation, operation: string): Promise<CreateServerResult> {
+    const { name, port, plutoKey, mapEntry } = prepared;
     deploymentStage(operation, 'Preparing server files');
     const hostDir = `${config.serversRoot}/${name}`;
     const initialConfig = getPromotedServerConfig('t5', mapEntry.canonical);
@@ -304,7 +320,7 @@ async function createServerUnlocked(input: CreateServerInput, begin: (name: stri
     // now instead of on the next 15s tick.
     void kick().catch((err) => console.error('[createServer] post-create kick failed:', err));
 
-    return { ok: true, name, port };
+    return { ok: true, name, port, deploymentId: operation };
 }
 
 /**
@@ -453,5 +469,75 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
         };
     }
 
+    const unfinished = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' AND state = 'awaiting-health'").all(name) as { id: string }[];
+    for (const { id } of unfinished) finishDeployment(id, { ok: false, message: 'Server destroyed before creation health checks completed' });
     return { ok: true, warnings };
+}␍interface CreationJob {
+    deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
+}
+
+async function provisionCreation(id: string): Promise<CreateServerResult> {
+    const job = db().prepare('SELECT j.*, d.triggered_by FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE j.deployment_id = ?').get(id) as CreationJob;
+    creating = true;
+    db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(id);
+    try {
+        const mapEntry = t5ZombieMapBySlug(job.map_slug);
+        const plutoKey = getServerKey(job.server);
+        if (!mapEntry || !plutoKey) {
+            const result = { ok: false as const, kind: 'bad-request' as const, message: 'Saved server key or selected map is no longer available' };
+            finishDeployment(id, result);
+            return result;
+        }
+        const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by },
+            { ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
+        if (!result.ok) finishDeployment(id, result);
+        else {
+            db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
+            deploymentStage(id, 'Waiting for Komodo health');
+        }
+        return result;
+    } catch (err) {
+        finishDeployment(id, { ok: false, message: (err as Error).message });
+        throw err;
+    } finally { creating = false; }
+}
+
+let creationTick: Promise<void> | null = null;
+export function processCreations(): Promise<void> {
+    if (creationTick) return creationTick;
+    creationTick = runCreations().finally(() => { creationTick = null; });
+    return creationTick;
+}
+
+async function runCreations(): Promise<void> {
+    if (creating || destroying.size > 0) return;
+    const queued = db().prepare("SELECT id FROM server_deployments WHERE action = 'create' AND state = 'queued' ORDER BY started_at").all() as { id: string }[];
+    for (const { id } of queued) {
+        if (creating || destroying.size > 0) break;
+        await provisionCreation(id).catch(err => console.error('[creation] provisioning failed:', err));
+    }
+    const checking = db().prepare("SELECT j.* FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state = 'awaiting-health'").all() as CreationJob[];
+    for (const job of checking) {
+        const id = job.deployment_id;
+        if (!getServer(job.server)) {
+            finishDeployment(id, { ok: false, message: 'Server was removed before creation health checks completed' });
+            continue;
+        }
+        if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
+        try {
+            const komodo = await komodoInspectHealthy(job.server);
+            if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
+            const rcon = await rconStatusResult(job.server);
+            if (!rcon.ok) { deploymentStage(id, 'Waiting for RCON health: ' + rcon.reason); continue; }
+            const actualMap = rcon.status.map;
+            if (actualMap !== job.expected_map) {
+                deploymentStage(id, 'Waiting for selected map: expected ' + job.expected_map + ', RCON reports ' + (actualMap ?? 'no map'));
+                continue;
+            }
+            deploymentStage(id, 'Verified Komodo healthy, RCON healthy, and map ' + actualMap);
+            finishDeployment(id, { ok: true });
+        } catch (err) {
+            deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message);
+        }
+    }
 }

src/lib/server/fleet/deployments.ts

@@ -21,8 +21,8 @@ export function recordDeploymentEvent(id: string, message: string): void {
 }
 
 export function getDeployment(id: string) {
-    const deployment = db().prepare(`SELECT d.*, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
-        FROM server_deployments d WHERE d.id = ?`).get(id);
+    const deployment = db().prepare(`SELECT d.*, j.expected_map, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
+        FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id);
     if (!deployment) return null;
     const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id);
     return { deployment, events };

src/lib/server/integrations/komodo.ts

@@ -277,7 +277,7 @@ const STATE_KEYS = new Set([
     'docker_status'
 ]);
 
-function extractContainerState(payload: unknown): string | null {
+export function extractContainerState(payload: unknown): string | null {
     const seen = new Set<unknown>();
     let found: string | null = null;
 
@@ -297,6 +297,12 @@ function extractContainerState(payload: unknown): string | null {
             const status = so['Status'];
             const running = so['Running'];
             const health = so['Health'];
+            // A running process can still be starting or unhealthy. Honor an
+            // explicit Docker health check before accepting its running state.
+            if ((running === true || status === 'running') && health && typeof health === 'object') {
+                const hs = (health as Record<string, unknown>)['Status'];
+                if (typeof hs === 'string' && hs.toLowerCase() !== 'healthy') { found = hs; return; }
+            }
             if (typeof status === 'string') { found = status; return; }
             if (running === true) { found = 'running'; return; }
             if (health && typeof health === 'object') {

src/lib/server/pipeline/worker.ts

@@ -1,4 +1,4 @@
-import { processDestructions } from '../fleet/create.js';
+import { processDestructions, processCreations } from '../fleet/create.js';
 import { sweep } from './sweep.js';
 import { betaTick } from './beta-runner.js';
 import { emit } from './bus.js';
@@ -28,6 +28,8 @@ function tick(): Promise<void> {
 }
 
 async function runTick(): Promise<void> {
+    try { await processCreations(); }
+    catch (err) { console.error('[worker] creation error:', err); }
     try { await processDestructions(); }
     catch (err) { console.error('[worker] destruction error:', err); }
     try {

src/lib/ui/components/DeployServerDialog.svelte

@@ -66,7 +66,8 @@
             const [keysRes, serversRes] = await Promise.all([fetch('/api/server-keys'), fetch('/api/servers')]);
             if (!keysRes.ok || !serversRes.ok) throw new Error('Could not load saved keys and server slots. Reopen this dialog to retry.');
             savedIds = (await keysRes.json()).keys.map((k: { server_id: string }) => k.server_id);
-            deployedIds = (await serversRes.json()).servers.map((s: { name: string }) => s.name);
+            const fleet = await serversRes.json();
+            deployedIds = [...fleet.servers.map((s: { name: string }) => s.name), ...(fleet.reservedNames ?? [])];
             keysLoaded = true;
         } catch (err) { flash = (err as Error).message; }
     }
@@ -100,15 +101,15 @@
                 headers: { 'Content-Type': 'application/json' },
                 body: JSON.stringify({ gameMode, mapSlug: selectedMap.slug, expectedName: previewName, plutoKey: plutoKey.trim() || undefined })
             });
-            const body = (await res.json()) as { ok: boolean; name?: string; port?: number; error?: string; kind?: string };
-            if (!body.ok || !body.name) {
+            const body = (await res.json()) as { ok: boolean; name?: string; port?: number; deploymentId?: string; error?: string; kind?: string };
+            if (!res.ok || !body.ok || !body.deploymentId) {
                 flash = body.error ?? 'Deploy failed';
                 if (body.kind === 'no-slot') await loadKeys();
                 deploying = false;
                 return;
             }
             close();
-            await goto(`/servers/${encodeURIComponent(body.name)}`);
+            await goto(`/deployments/${encodeURIComponent(body.deploymentId)}`);
         } catch (err) {
             flash = (err as Error).message;
             deploying = false;