From f171889dfb04ec9b2a3f3b6f7374ba1d37b1a8de Mon Sep 17 00:00:00 2001 From: josh Date: Sat, 19 Sep 2026 20:33:08 -0400 Subject: [PATCH] Pace collection startups and clarify deployment queue progress --- .env.example | 4 + README.md | 12 +- docker-compose.yml | 1 + docs/deployment-blueprints.md | 38 +- src/lib/deployment-plan.ts | 7 +- src/lib/server/config.ts | 1 + src/lib/server/fleet/collections.ts | 14 +- src/lib/server/fleet/create.ts | 101 ++++- src/lib/server/integrations/komodo.ts | 5 +- .../components/DeployCollectionDialog.svelte | 35 +- src/routes/deployments/+page.svelte | 10 +- src/routes/deployments/[id]/+page.svelte | 2 +- .../deployments/collections/[id]/+page.svelte | 107 +++++- tests/unit/create-server.test.ts | 33 +- tests/unit/creation-queue.test.ts | 357 ++++++++++++++++++ tests/unit/deployment-collections.test.ts | 16 +- tests/unit/komodo-write.test.ts | 29 ++ tests/unit/launch-recipes.test.ts | 2 + 18 files changed, 693 insertions(+), 81 deletions(-) create mode 100644 tests/unit/creation-queue.test.ts create mode 100644 tests/unit/komodo-write.test.ts diff --git a/.env.example b/.env.example index 250bbaf..f19f8da 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,10 @@ DEPLOY_KEEP_DAYS=7 # A slot stays occupied until Komodo and RCON confirm recovery. DEPLOYMENT_RESTART_CONCURRENCY=1 +# New-server startup limit, shared by all collections and single-server launches. +# A slot stays occupied through Komodo, RCON, and selected-map verification. +DEPLOYMENT_CREATE_CONCURRENCY=1 + # Komodo KOMODO_ADDRESS=https://komodo.example.internal KOMODO_API_KEY= diff --git a/README.md b/README.md index 9502433..7f09a4c 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,8 @@ may have accepted the request. Slots persist in SQLite across app restarts, and overlapping timer/API triggers share the active run. Beta options to skip health or match gates still work; skipping a beta gate does not release the fleet's recovery slot early. New-server -creation is separate from deployment restarts and still starts its container -directly. +creation uses a separate shared startup queue, holding its slot through container, +RCON, and selected-map verification. No token, no session, no auth freeze. If Komodo or RCON aren't configured, the sweep marks pending rows `manual` so nothing hangs forever. @@ -114,6 +114,14 @@ shows target/existing/missing counts and IDs, ports, and key requirements for ne servers only. Already-satisfied targets create nothing; excess servers are kept. All jobs and capacity reservations are committed together before provisioning; one collection page tracks each server's independent progress and failures. +New servers start one at a time by default across all collections and individual +launches. A startup slot stays occupied until Komodo and RCON are healthy and +RCON reports the selected map; an unready server holds the queue until it recovers +or is removed. The queue survives application restarts. The collection preview +shows the startup limit, and its progress page shows active startups and each +server's place in the shared queue. Set `DEPLOYMENT_CREATE_CONCURRENCY` to a +positive integer to increase the limit. Each worker pass starts at most the +available number of slots; immediate failures do not drain the remaining queue. Existing single-server blueprints can be expanded with **Edit**. Single-server launches and launching again from recorded recipes remain available. Blueprints save maps and target lobby counts. Every new server uses the latest diff --git a/docker-compose.yml b/docker-compose.yml index 2419465..b50ba84 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,7 @@ services: DB_PATH: /data/cherry.db DEPLOY_KEEP_DAYS: "7" DEPLOYMENT_RESTART_CONCURRENCY: ${DEPLOYMENT_RESTART_CONCURRENCY:-1} + DEPLOYMENT_CREATE_CONCURRENCY: ${DEPLOYMENT_CREATE_CONCURRENCY:-1} KOMODO_ADDRESS: ${KOMODO_ADDRESS} KOMODO_API_KEY: ${KOMODO_API_KEY} KOMODO_API_SECRET: ${KOMODO_API_SECRET} diff --git a/docs/deployment-blueprints.md b/docs/deployment-blueprints.md index f92fca9..78b2423 100644 --- a/docs/deployment-blueprints.md +++ b/docs/deployment-blueprints.md @@ -28,7 +28,7 @@ destroy servers or change existing servers' files. 3. Choose **Save & preview** to review every new server ID and game port. 4. Enter each lobby's Plutonium key, or leave its field blank to reuse the key saved for that exact server ID. -5. Choose **Create N missing servers** and follow their collection progress page. +5. Choose **Queue N servers** and follow their collection progress page. If every target is already met, choose **Done**; no operation is created. Blueprints support up to 40 distinct game/map selections, 1–20 lobbies per map, @@ -58,11 +58,24 @@ capacity changes before submission, the request is rejected without changes and requires a fresh preview. Previously saved additive previews are checked against the current shortfall too. -The worker provisions servers sequentially; health verification and failures are -recorded separately for each. One failure does not undo servers already created -or prevent the rest of the queued collection from being attempted. The collection -page summarizes verified, active, and failed operations and links to every -server's timeline and recovery options. Each child links back to its collection. +All collections and individual launches share one startup queue, ordered by request +time. By default, one server starts at a time. Its startup slot remains occupied +until Komodo is healthy, RCON answers, and the reported map matches the selection. +Only then can the next queued server start. Set `DEPLOYMENT_CREATE_CONCURRENCY` to +a positive integer to allow more simultaneous startups. Each worker pass admits +at most the available slots, even if an attempt fails immediately. + +Names, ports, and queue order survive application restarts. A server still booting +or waiting for health keeps its slot until it passes verification or is removed. +A lost startup response also keeps the slot because the container may already be +running. A confirmed creation failure is recorded for review, allowing later +queued servers to proceed on subsequent passes without undoing successful launches. + +The collection page separates ready, launching, queued, and failed servers. It +highlights the current startup, shows each waiting server's queue position, and +links to earlier launches when another collection occupies the available capacity. +Every server links to its operation timeline and recovery options, and each child +operation links back to its collection. ## Create and review a launch @@ -148,6 +161,9 @@ when the original server is still registered on its assigned port and no later lifecycle operation has targeted that ID. It resumes observation only; it never recreates or restarts a container. Missing servers and later destruction attempts require inspection rather than replaying provisioning automatically. +After an application restart, the worker automatically resumes observation of +eligible interrupted startups so their occupied slots are retained. It never +repeats container creation or deployment as part of this recovery. ## API @@ -158,13 +174,17 @@ require inspection rather than replaying provisioning automatically. - `PUT /api/blueprints/:id`: edit `{ name, description?, entries }`. All entries use the latest promoted setup when launched. - `DELETE /api/blueprints/:id`: remove a library entry. -- `POST /api/deployment-collections/preview`: preview `{ blueprintId }`. +- `POST /api/deployment-collections/preview`: preview `{ blueprintId }`, including + the configured `launchConcurrency`. - `POST /api/deployment-collections`: atomically queue `{ previewId, keys? }`, where `keys` maps exact preview server IDs to optional replacement keys. A satisfied preview returns `{ ok: true, collectionId: null }` with HTTP 200, without creating a collection, saving keys, or kicking the worker. -- `GET /api/deployment-collections`: list collection progress summaries. -- `GET /api/deployment-collections/:id`: collection and per-server progress. +- `GET /api/deployment-collections`: list collection progress summaries with + separate `queued`, `provisioning`, and `verifying` counts. +- `GET /api/deployment-collections/:id`: collection and per-server progress, + each waiting server's global `queuePosition`, and the shared queue's concurrency, + active startups, and queued count. - `POST /api/deployments/preview`: preview `{ game, mapSlug }` or `{ deploymentId }` for a new single server using the latest promoted setup. All blueprints, including one-server targets, must use the collection endpoint so existing servers count correctly. diff --git a/src/lib/deployment-plan.ts b/src/lib/deployment-plan.ts index 7e190cf..05f2618 100644 --- a/src/lib/deployment-plan.ts +++ b/src/lib/deployment-plan.ts @@ -36,6 +36,7 @@ export interface CollectionPreview { id: string; expiresAt: string; blueprintName: string; + launchConcurrency: number; entries: { count: number; existing: number; missing: number; existingServers: string[]; recipe: LaunchRecipeSummary }[]; servers: { entryIndex: number; name: string | null; port: number | null; hasSavedKey: boolean }[]; blockers: string[]; @@ -50,12 +51,16 @@ export interface CollectionSummary { done: number; failed: number; active: number; + queued: number; + provisioning: number; + verifying: number; } export interface CollectionDetail { collection: CollectionSummary; + queue: { concurrency: number; active: { id: string; server: string; state: string; stage: string }[]; queued: number }; servers: { id: string; server: string; port: number; map_slug: string; state: string; stage: string; - started_at: string; finished_at: string | null; error: string | null; server_exists: number }[]; + started_at: string; finished_at: string | null; error: string | null; server_exists: number; queuePosition: number | null }[]; } export interface LaunchPreview { diff --git a/src/lib/server/config.ts b/src/lib/server/config.ts index fc770a4..2a81a72 100644 --- a/src/lib/server/config.ts +++ b/src/lib/server/config.ts @@ -40,6 +40,7 @@ export const config = { dbPath: req('DB_PATH', './data/cherry.db'), deployKeepDays: num('DEPLOY_KEEP_DAYS', 7), restartConcurrency: positiveInt('DEPLOYMENT_RESTART_CONCURRENCY', 1), + createConcurrency: positiveInt('DEPLOYMENT_CREATE_CONCURRENCY', 1), httpTimeoutMs: num('HTTP_TIMEOUT_MS', 10_000), komodo: { address: env.KOMODO_ADDRESS ?? '', diff --git a/src/lib/server/fleet/collections.ts b/src/lib/server/fleet/collections.ts index b8a3e9f..9434686 100644 --- a/src/lib/server/fleet/collections.ts +++ b/src/lib/server/fleet/collections.ts @@ -3,7 +3,7 @@ import type { CollectionDetail, CollectionPreview, CollectionSummary } from '$li import { db } from '../db/index.js'; import { config, komodoConfigured, rconConfigured } from '../config.js'; import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js'; -import { nextAvailablePort, nextLobbyNumber, queueCreateServer } from './create.js'; +import { creationQueue, nextAvailablePort, nextLobbyNumber, queueCreateServer } from './create.js'; import { getServerKey } from './server-keys.js'; import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js'; @@ -70,7 +70,7 @@ export function previewCollection(blueprintId: unknown): CollectionPreview { db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString()); db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers) VALUES (?, ?, ?, ?, ?)') .run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers)); - return { id, expiresAt, blueprintName, servers, blockers, ready: blockers.length === 0, + return { id, expiresAt, blueprintName, launchConcurrency: config.createConcurrency, servers, blockers, ready: blockers.length === 0, entries: entries.map((entry, index) => ({ count: entry.count, ...counts[index], recipe: summarizeRecipe(entry.recipe) })) }; })(); } @@ -123,7 +123,10 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): const summarySql = `SELECT c.id, c.name, c.created_at, COUNT(d.id) AS total, COALESCE(SUM(d.state IN ('done', 'warning')), 0) AS done, COALESCE(SUM(d.state IN ('failed', 'interrupted')), 0) AS failed, - COALESCE(SUM(d.state IN ('queued', 'running', 'awaiting-health', 'waiting')), 0) AS active + COALESCE(SUM(d.state IN ('queued', 'running', 'awaiting-health', 'waiting')), 0) AS active, + COALESCE(SUM(d.state = 'queued'), 0) AS queued, + COALESCE(SUM(d.state = 'running'), 0) AS provisioning, + COALESCE(SUM(d.state = 'awaiting-health'), 0) AS verifying FROM server_launch_collections c LEFT JOIN server_collection_members m ON m.collection_id = c.id LEFT JOIN server_deployments d ON d.id = m.deployment_id`; @@ -139,5 +142,8 @@ export function getCollection(id: string): CollectionDetail | null { EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists FROM server_collection_members m JOIN server_deployments d ON d.id = m.deployment_id JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY m.position`).all(id) as CollectionDetail['servers']; - return { collection, servers }; + const queue = creationQueue(); + const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1])); + return { collection, queue: { concurrency: config.createConcurrency, active: queue.active, queued: queue.queued.length }, + servers: servers.map(server => ({ ...server, queuePosition: positions.get(server.id) ?? null })) }; } diff --git a/src/lib/server/fleet/create.ts b/src/lib/server/fleet/create.ts index 6e45849..9a980bf 100644 --- a/src/lib/server/fleet/create.ts +++ b/src/lib/server/fleet/create.ts @@ -1,5 +1,5 @@ import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js'; -import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER, resumeCleanup } from './deployments.js'; +import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification } from './deployments.js'; import { randomUUID } from 'node:crypto'; import type { LaunchPreview } from '$lib/deployment-plan.js'; import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from './launch-recipes.js'; @@ -234,14 +234,17 @@ function nextName(input: CreateServerInput): string | null { return lobby === null ? null : `${base}_${lobby}`; } -// Await provisioning for internal callers; completion still requires worker health verification. +// Internal callers use the same durable queue and capacity limit as HTTP launches. +// Await this pass's provisioning when possible; a busy host leaves the job queued. export async function createServer(input: CreateServerInput): Promise { - if (creating || destroying.size > 0) return { ok: false, kind: 'no-slot', message: 'Another server operation is in progress. Please retry shortly.' }; const queued = queueCreateServer(input); if (!queued.ok) return queued; const existing = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(queued.deploymentId) as { state: string }; if (existing.state !== 'queued') return queued; - return await provisionCreation(queued.deploymentId); + const outcomes = await creationPass(); + const outcome = outcomes.get(queued.deploymentId); + if (outcome instanceof Error) throw outcome; + return outcome ?? queued; } function prepareCreation(input: CreateServerInput): PreparedCreation | Extract { @@ -378,6 +381,12 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared deploymentStage(operation, 'Starting container'); const deployRes = await komodoDeploy(name); if (!deployRes.ok) { + if (deployRes.kind === 'unreachable' || (deployRes.httpStatus !== undefined && deployRes.httpStatus >= 500)) { + // A timeout or server error may arrive after the remote boot began. + // Preserve files, identity, and capacity until readiness is known. + recordDeploymentEvent(operation, 'Startup response uncertain; verifying the existing server: ' + deployRes.message); + return { ok: true, name, port, deploymentId: operation }; + } // Container never started — undo everything so retrying is clean. await komodoDeleteDeployment(name).catch(() => {}); db().prepare('DELETE FROM servers WHERE name = ?').run(name); @@ -596,30 +605,71 @@ async function provisionCreation(id: string): Promise { } return result; } catch (err) { - finishDeployment(id, { ok: false, message: (err as Error).message }); + const server = getServer(job.server); + if (server?.game_port === job.port) { + // A lost response may have happened after Komodo started the server. + // Keep its slot and observe readiness instead of starting another boot. + db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id); + recordDeploymentEvent(id, 'Startup response interrupted: ' + (err as Error).message); + deploymentStage(id, 'Verifying existing server after interrupted startup'); + } else finishDeployment(id, { ok: false, message: (err as Error).message }); throw err; } finally { creating = false; } } -let creationTick: Promise | null = null; -export function processCreations(): Promise { +type CreationOutcomes = Map; +let creationTick: Promise | null = null; +function creationPass(): Promise { if (creationTick) return creationTick; creationTick = runCreations().finally(() => { creationTick = null; }); return creationTick; } -async function runCreations(): Promise { - if (creating || destroying.size > 0) return; - const queued = db().prepare("SELECT id FROM server_deployments WHERE action = 'create' AND state = 'queued' ORDER BY started_at").all() as { id: string }[]; - for (const { id } of queued) { - if (creating || destroying.size > 0) break; - await provisionCreation(id).catch(err => console.error('[creation] provisioning failed:', err)); +export async function processCreations(): Promise { + await creationPass(); +} + +/** One host-wide FIFO, shared by collections and individual launches. */ +export function creationQueue() { + type Entry = { id: string; server: string; state: string; stage: string }; + const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments + WHERE action = 'create' AND state IN ('queued', 'running', 'awaiting-health') + ORDER BY started_at, rowid`).all() as Entry[]; + return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') }; +} + +async function runCreations(): Promise { + const outcomes: CreationOutcomes = new Map(); + if (creating || destroying.size > 0) return outcomes; + // Restart recovery is observation-only: never repeat external creation or + // deployment. A known interrupted startup must retain its readiness gate. + const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action = 'create' AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[]; + for (const { id } of interrupted) { + if (resumeVerification(id)) continue; + // A queued removal does not make the interrupted container disappear. + // Keep its slot until removal finishes, without resuming normal work or + // confusing it with a newer creation of the same server identity. + const removing = db().prepare(`SELECT 1 FROM server_deployments d + JOIN server_creation_jobs j ON j.deployment_id = d.id + JOIN servers s ON s.name = j.server AND s.game_port = j.port + WHERE d.id = ? AND EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server + AND newer.rowid > d.rowid AND newer.action = 'destroy' AND newer.state IN ('waiting', 'running')) + AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server + AND newer.rowid > d.rowid AND newer.action = 'create')`).get(id); + if (removing) { + db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id); + deploymentStage(id, 'Waiting for requested server removal after interrupted startup'); + } } - const checking = db().prepare("SELECT j.* FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state = 'awaiting-health'").all() as CreationJob[]; + + // Check only previously started servers. Newly launched servers keep their + // slot until a later pass, even if a health probe would answer immediately. + const checking = db().prepare(`SELECT j.* FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id + WHERE d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[]; for (const job of checking) { const id = job.deployment_id; - if (!getServer(job.server)) { - finishDeployment(id, { ok: false, message: 'Server was removed before creation health checks completed' }); + if (getServer(job.server)?.game_port !== job.port) { + finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' }); continue; } if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue; @@ -639,4 +689,23 @@ async function runCreations(): Promise { deploymentStage(id, 'Waiting for health verification: ' + (err as Error).message); } } + + // Snapshot the budget once. Fast success/failure must not turn a single + // worker pass into an unbounded stream of expensive container startups. + const queue = creationQueue(); + const available = Math.max(0, config.createConcurrency - queue.active.length); + for (const { id } of queue.queued.slice(0, available)) { + if (creating || destroying.size > 0) break; + // API calls can cancel a selected job or resume an older startup while + // a previous provision awaits Komodo. Recheck without expanding this pass. + if (creationQueue().active.length >= config.createConcurrency) break; + if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action = 'create' AND state = 'queued'").get(id)) continue; + try { outcomes.set(id, await provisionCreation(id)); } + catch (err) { + const failure = err instanceof Error ? err : new Error(String(err)); + outcomes.set(id, failure); + console.error('[creation] provisioning failed:', failure); + } + } + return outcomes; } diff --git a/src/lib/server/integrations/komodo.ts b/src/lib/server/integrations/komodo.ts index 35f46a9..afaee6f 100644 --- a/src/lib/server/integrations/komodo.ts +++ b/src/lib/server/integrations/komodo.ts @@ -429,7 +429,7 @@ export async function komodoInspectHealthy(serverName: string): Promise { if (isDialogBackdrop(event, dialog) && !busy) open = false; }} oncancel={event => { if (busy) event.preventDefault(); }} onclose={() => open = false}>
-
Your lineup, ready to launch

{preview?.blueprintName ?? 'Preparing your collection'}

+
Collection launch

{preview?.blueprintName ?? 'Preparing your collection'}

{#if error}{/if} {#if !preview}

{busy ? 'Finding a lobby and port for every server…' : 'Refresh to prepare another preview.'}

{#if !busy}{/if} {:else} @@ -57,25 +58,30 @@ {#each preview.entries as entry}{/each}
MapTargetExisting / queuedTo create
{entry.recipe.mapName} {entry.recipe.game.toUpperCase()}{entry.count}{entry.existing}{entry.missing}

Existing and queued servers count toward each map’s target, including stopped servers. Extra servers are kept. Servers awaiting removal do not count.

-

{preview.servers.length ? 'Review the missing lobbies' : 'Nothing to create'}

+ {#if preview.servers.length} +
+ +
{preview.launchConcurrency === 1 ? 'A steady launch, one server at a time' : `A steady launch, up to ${preview.launchConcurrency} servers at a time`}

{preview.launchConcurrency === 1 ? 'Each server finishes its startup checks before the next begins.' : 'New servers start as earlier launches finish their startup checks.'} Other launches share this queue, keeping startup load under control.

+
+ {/if} +

{preview.servers.length ? 'Your launch order' : 'Nothing to create'}

{#each preview.blockers as blocker}{/each} {#if preview.servers.length} -

Only the missing servers below need keys. Saved keys are reused for these exact IDs. Fleet counts and slots are checked again before anything is queued. Each new server uses the latest promoted setup when provisioning begins.

+

Add a key for each new lobby below, or reuse its saved key. Fleet counts and available slots are checked again when you queue the launch.

{ event.preventDefault(); void launch(); }}> -
+
    {#each preview.servers as server, index} -
    -
    {preview.entries[server.entryIndex].recipe.mapName} · {server.port === null ? 'No available port' : `Port ${server.port}`}{server.name ?? `No available lobby (${index + 1})`}
    +
  1. + +
    {#if server.name}{:else}{preview.entries[server.entryIndex].recipe.mapName}{/if}{server.name ?? 'No available lobby'} · {server.port === null ? 'No available port' : `Port ${server.port}`}
    {#if server.name}{/if} -
    2. + {/each} -
-

The host must have the selected games and DLC. This preview checks settings and capacity; it does not probe game files, images, credentials, or network access.

-
-

Your collection is queued in one action. Servers provision in sequence and verify independently. Preview expires at {new Date(preview.expiresAt).toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' })}.

+ +

Follow every server from queued to ready.
Preview expires at {new Date(preview.expiresAt).toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' })}.

{:else}

Your fleet already meets or exceeds every target in this blueprint. No servers will be created or removed.

{/if} -
Review the current promoted setup for each map{#each preview.entries as entry}
{/each}
+
Setup and launch requirements

Each server uses the latest promoted setup when its turn begins. The host must have the selected games and DLC. This preview checks settings and capacity; it does not probe game files, images, credentials, or network access.

{#each preview.entries as entry}
{/each}
{/if} @@ -84,6 +90,7 @@ dialog { width: min(820px, calc(100vw - 32px)); max-height: calc(100dvh - 40px); padding: 0; border: 1px solid var(--border-strong); border-radius: var(--radius-lg); background: var(--surface); color: var(--text-primary); }dialog::backdrop { background: rgb(0 0 0 / .7); backdrop-filter: blur(5px); }.composer { display: grid; gap: 20px; padding: 28px; } header, .section-title, .actions { display: flex; align-items: center; justify-content: space-between; gap: 12px; flex-wrap: wrap; }h2, h3, p { margin: 0; }h2 { font-size: 25px; overflow-wrap: anywhere; }h3 { font-size: 16px; }.eyebrow { display: block; font-size: var(--text-xs); color: var(--text-secondary); text-transform: uppercase; letter-spacing: .1em; margin-bottom: 7px; }.close { border: 0; background: none; color: var(--text-secondary); font-size: 26px; cursor: pointer; }.muted, .caption { font-size: 13px; color: var(--text-secondary); line-height: 1.6; }.caption { font-size: var(--text-xs); }.error { color: var(--state-danger); font-size: 13px; line-height: 1.6; } .lineup { background: var(--surface-sunken); border: 1px solid var(--border-subtle); border-radius: var(--radius-md); padding: 20px; }.lineup > strong { font-size: 18px; }.quantity { color: var(--accent-primary-hover); font-weight: 600; }.lineup small { color: var(--text-secondary); }.lineup table { width: 100%; margin: 16px 0; border-collapse: collapse; font-size: 12px; table-layout: fixed; }.lineup th, .lineup td { text-align: center; padding: 10px 4px; border-bottom: 1px solid var(--border-subtle); overflow-wrap: anywhere; }.lineup th:first-child { text-align: left; width: 34%; }.lineup thead { color: var(--text-secondary); font-size: var(--text-xs); } - form, .servers { display: grid; gap: 16px; }.server-row { display: grid; grid-template-columns: 1fr 240px; align-items: center; gap: 20px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); padding: 16px; }.server-row > div { min-width: 0; }.server-row strong { font-family: var(--font-mono); font-size: 13px; overflow-wrap: anywhere; }label { display: grid; gap: 7px; font-size: var(--text-xs); color: var(--text-secondary); }input { width: 100%; box-sizing: border-box; border: 1px solid var(--border-strong); border-radius: 7px; background: var(--surface-sunken); color: var(--text-primary); padding: 10px; font-size: 13px; }.actions { justify-content: flex-end; }.text-button { border: 0; background: none; color: var(--accent-primary-hover); cursor: pointer; font-size: 12px; }.recipes { border-top: 1px solid var(--border-subtle); padding-top: 20px; }.recipes summary { cursor: pointer; font-size: 13px; }.recipe { padding: 20px 0; border-bottom: 1px solid var(--border-subtle); } - @media (max-width: 600px) { .composer { padding: 20px; }.server-row { grid-template-columns: 1fr; gap: 12px; } } + form { display: grid; gap: 20px; }.servers { margin: 0; padding: 0; list-style: none; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); overflow: hidden; }.server-row { display: grid; grid-template-columns: 28px minmax(0, 1fr) 220px; align-items: center; gap: 16px; padding: 20px 16px; }.server-row + .server-row { border-top: 1px solid var(--border-subtle); }.order { font-family: var(--font-mono); color: var(--text-tertiary); font-size: 12px; align-self: start; padding-top: 3px; }.server-identity { display: grid; gap: 8px; min-width: 0; }.server-row strong { font-size: 14px; overflow-wrap: anywhere; }.server-id { font-family: var(--font-mono); font-size: 11px; line-height: 1.6; color: var(--text-secondary); overflow-wrap: anywhere; }label { display: grid; gap: 7px; font-size: var(--text-xs); color: var(--text-secondary); }input { width: 100%; box-sizing: border-box; border: 1px solid var(--border-strong); border-radius: 7px; background: var(--surface-sunken); color: var(--text-primary); padding: 10px; font-size: 13px; }.actions { justify-content: flex-end; }.text-button { border: 0; background: none; color: var(--accent-primary-hover); cursor: pointer; font-size: 12px; }.recipes { border-top: 1px solid var(--border-subtle); padding-top: 20px; }.recipes summary { cursor: pointer; font-size: 13px; }.recipe { padding: 20px 0; border-bottom: 1px solid var(--border-subtle); } + .pacing-note { display: flex; align-items: flex-start; gap: 14px; background: color-mix(in srgb, var(--accent-primary) 8%, transparent); border-left: 2px solid var(--accent-primary); padding: 16px; border-radius: 0 var(--radius-sm) var(--radius-sm) 0; }.pacing-note svg { flex: 0 0 24px; width: 24px; height: 24px; color: var(--accent-primary-hover); }.pacing-note strong { font-size: 14px; }.pacing-note p { margin-top: 6px; color: var(--text-secondary); font-size: 12px; line-height: 1.6; }.launch-footer { display: flex; justify-content: space-between; align-items: center; gap: 20px; flex-wrap: wrap; }.requirements { margin-top: 16px; }button:focus-visible, input:focus-visible, summary:focus-visible { outline: 2px solid var(--border-focus); outline-offset: 3px; } + @media (max-width: 600px) { .composer { padding: 20px; }.lineup { padding: 16px; }.server-row { grid-template-columns: 24px minmax(0, 1fr); gap: 12px; padding: 16px; }.server-row label { grid-column: 2; }.launch-footer { align-items: stretch; flex-direction: column; gap: 16px; }.actions { justify-content: space-between; }h2 { font-size: 22px; } } diff --git a/src/routes/deployments/+page.svelte b/src/routes/deployments/+page.svelte index 01aa4f4..d5b486f 100644 --- a/src/routes/deployments/+page.svelte +++ b/src/routes/deployments/+page.svelte @@ -145,10 +145,11 @@

Collection launches

{#each visibleCollections.slice(0,collectionLimit) as collection (collection.id)} - -
{collection.name}{collection.active ? 'In progress' : collection.failed ? 'Needs attention' : 'Completed'}
-

{collection.done} of {collection.total} servers verified{collection.failed ? ` · ${collection.failed} need attention` : ''}{collection.active ? ` · ${collection.active} in progress` : ''}

- Started {fmt(collection.created_at)} + +
{collection.name}{collection.active ? collection.provisioning + collection.verifying ? 'Launching' : 'Queued' : collection.failed ? 'Needs attention' : 'Completed'}
+
{collection.done} / {collection.total} ready{#if collection.provisioning + collection.verifying}{collection.provisioning + collection.verifying} launching{/if}{#if collection.queued}{collection.queued} queued{/if}{#if collection.failed}{collection.failed} need attention{/if}
+ +
Queued {fmt(collection.created_at)}{collection.active ? 'Follow launch' : 'View results'} →
 {/each}
@@ -212,6 +213,7 @@ .deployment-row { display: block; text-decoration: none; color: inherit; padding: 20px; border: 1px solid var(--border-subtle); border-radius: var(--radius-md); background: var(--surface); overflow-wrap: anywhere; } .deployment-row:hover { border-color: var(--border-strong); background: var(--surface-raised); } .deployment-row:focus-visible { outline: 2px solid var(--border-focus); outline-offset: 3px; } + .collection-row { display: grid; gap: 16px; }.collection-counts { display: flex; flex-wrap: wrap; gap: 10px 20px; font-size: 13px; color: var(--text-secondary); }.collection-counts strong { color: var(--state-online); font-weight: 600; font-variant-numeric: tabular-nums; }.collection-meter { display: flex; height: 4px; background: var(--surface-raised); overflow: hidden; border-radius: var(--radius-pill); }.collection-meter .verified { background: var(--state-online); }.collection-meter .unsuccessful { background: var(--state-danger); }.collection-footer { display: flex; align-items: baseline; justify-content: space-between; flex-wrap: wrap; gap: 8px 16px; }.collection-footer .meta, .collection-link { font-size: 12px; }.collection-link { color: var(--accent-primary-hover); } .server { margin-left: 12px; font-family: var(--font-mono); font-size: 14px; } .meta { font-size: 13px; color: var(--text-secondary); } .error { color: var(--state-danger); } diff --git a/src/routes/deployments/[id]/+page.svelte b/src/routes/deployments/[id]/+page.svelte index a48a4bc..767e125 100644 --- a/src/routes/deployments/[id]/+page.svelte +++ b/src/routes/deployments/[id]/+page.svelte @@ -55,7 +55,7 @@ if (deployment.state === 'failed' || deployment.state === 'interrupted') return detail?.canResumeVerification ? 'The server is still registered. Inspect its status, then resume health verification without creating or restarting a container.' : 'Inspect the error and any remaining container or files before launching again. The activity below records how far this operation reached.'; - if (deployment.state === 'queued') return 'Your request is saved. Provisioning starts when the current server operation releases the worker.'; + if (deployment.state === 'queued') return 'Your request is saved in the launch queue. A startup slot opens after an earlier server passes its container, RCON, and map checks.'; if (deployment.state === 'waiting') return 'The server stays online while players finish. Removal begins only after a live check confirms it is empty.'; if (deployment.stage.startsWith('Waiting for selected map')) return 'Container and RCON checks passed. Waiting for RCON to report the selected engine map. Inspect the server configuration if this persists.'; if (deployment.stage.startsWith('Waiting for RCON')) return 'The container is healthy. Waiting for the game to answer RCON; check its startup logs, password, and network if this persists.'; diff --git a/src/routes/deployments/collections/[id]/+page.svelte b/src/routes/deployments/collections/[id]/+page.svelte index 98a2a63..a349ad7 100644 --- a/src/routes/deployments/collections/[id]/+page.svelte +++ b/src/routes/deployments/collections/[id]/+page.svelte @@ -2,18 +2,30 @@ import { onMount } from 'svelte'; import { pollVisible } from '$ui/poll.js'; import ServerIdentity from '$ui/components/ServerIdentity.svelte'; - import StatusBadge from '$ui/components/primitives/StatusBadge.svelte'; import { page } from '$app/stores'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte'; import Badge from '$ui/components/primitives/Badge.svelte'; import type { CollectionDetail } from '$lib/deployment-plan.js'; + type CollectionServer = CollectionDetail['servers'][number]; let detail = $state(null); let error = $state(''); let requestNumber = 0; let collection = $derived(detail?.collection); - const labels: Record = { queued: 'Queued', running: 'Provisioning', 'awaiting-health': 'Verifying health', done: 'Verified', warning: 'Completed with warnings', failed: 'Failed', interrupted: 'Interrupted' }; + const launching = (server: { state: string }) => ['running', 'awaiting-health'].includes(server.state); + const needsAttention = (server: { state: string }) => ['failed', 'interrupted', 'warning'].includes(server.state); + const currentServers = $derived(detail?.servers.filter(launching) ?? []); + const otherLaunches = $derived(detail?.queue.active.filter(active => !detail?.servers.some(server => server.id === active.id)) ?? []); + const firstQueued = $derived(detail?.servers.find(server => server.state === 'queued')); + const labels: Record = { queued: 'Queued', running: 'Launching', 'awaiting-health': 'Checking readiness', done: 'Ready', warning: 'Needs review', failed: 'Failed', interrupted: 'Interrupted' }; + + function queueHint(server: CollectionServer) { + if (server.queuePosition === 1) return 'Next in the launch queue. Starts when a place opens.'; + if (server.queuePosition) return `${server.queuePosition - 1} queued server${server.queuePosition === 2 ? '' : 's'} ahead. Starts automatically when it is this lobby’s turn.`; + return 'Waiting for its turn in the launch queue.'; + } + async function load() { const id = $page.params.id ?? ''; const current = ++requestNumber; @@ -30,30 +42,87 @@ {collection?.name ?? 'Collection launch'} | Electric Cherry
- + {#snippet crumbs()}{/snippet} {#if error}{/if} {#if collection && detail} -
-

{collection.active ? 'Your lineup is coming online' : collection.failed ? 'Some servers need attention' : 'Your lineup is ready'}

{collection.active ? 'In progress' : collection.failed ? 'Needs attention' : 'Completed'}
-
Servers
{collection.total}
Verified
{collection.done}
In progress
{collection.active}
Need attention
{collection.failed}
- -

Every server was queued together. Provisioning runs in sequence, then each server verifies its container, RCON, and map. Open a server’s operation for its full timeline{collection.failed ? ' and recovery options' : ''}.

+
+
+
Collection progress

{collection.active ? currentServers.length ? 'Bringing your lineup online' : 'Your lineup is queued' : collection.failed ? 'Launch complete with issues' : 'Your lineup is ready'}

+ {collection.done + collection.failed} / {collection.total} finished +
+
+ + +
+
+
Ready
{collection.done}
+
Launching
{collection.provisioning + collection.verifying}
+
Queued
{collection.queued}
+
0}>
Need attention
{collection.failed}
+
+

{collection.done} ready, {collection.provisioning + collection.verifying} launching, {collection.queued} queued, {collection.failed} need attention.

+
+ +

{#if collection.active}{detail.queue.concurrency === 1 ? 'One server at a time.' : `Up to ${detail.queue.concurrency} servers at a time.`} {detail.queue.concurrency === 1 ? 'Each finishes startup checks before the next begins.' : 'New servers start as earlier launches finish their startup checks.'} The queue is shared across all launches.{:else if collection.failed}The queue has finished. Open any server that needs attention to review its timeline and recovery options.{:else}Every server passed its startup checks. View your fleet →{/if}

+
-
- {#each detail.servers as server (server.id)} - -
-

{server.stage}

- {#if server.error}

{server.error}

{/if} -
Port {server.port} · {server.server.startsWith('t6_') ? 'Black Ops II' : 'Black Ops I'}View operation →
- - {/each} + + {#if currentServers.length} +
+ Launching now + {#each currentServers as server (server.id)} + + +

{server.state === 'awaiting-health' ? 'Checking the container, connection, and expected map.' : server.stage}

+ Follow progress + + {/each} +
+ {:else if collection.queued > 0} + + {/if} + +
+

Launch order

{collection.total} server{collection.total === 1 ? '' : 's'}
+
    + {#each detail.servers as server, index (server.id)} +
  1. + + +
    +
    {server.state === 'queued' && server.queuePosition === 1 ? 'Up next' : labels[server.state] ?? server.state}
    +

    {server.error ?? (server.state === 'queued' ? queueHint(server) : server.state === 'done' ? 'Startup checks passed.' : server.stage)}

    +
    Port {server.port}{needsAttention(server) ? 'Review & recover' : 'View operation'}
    +
    +     +
    2. + {/each} +
+

Readiness is verified at launch. The fleet view shows each server’s current health.

- {:else if !error}

Loading collection…

{/if} + {:else if !error}

Loading collection…

{/if}
diff --git a/tests/unit/create-server.test.ts b/tests/unit/create-server.test.ts index a2bf41d..10e73df 100644 --- a/tests/unit/create-server.test.ts +++ b/tests/unit/create-server.test.ts @@ -34,6 +34,13 @@ beforeEach(async () => { vi.stubEnv('KOMODO_API_SECRET', 'test'); vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host'); vi.stubEnv('RCON_PASSWORD', 'test-rcon'); + const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { rconStatusResult, rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(komodoInspectHealthy).mockReset().mockResolvedValue({ ok: true, state: 'running' }); + vi.mocked(rconStatusResult).mockReset().mockResolvedValue({ ok: true, + endpoint: { host: 'localhost', port: 27016 }, rawReply: '', + status: { players: 0, map: 'zombie_cosmodrome', gametype: null, roster: [] } }); + vi.mocked(rconGetPlayerCount).mockReset().mockResolvedValue(0); }); afterEach(async () => { @@ -48,7 +55,10 @@ describe('destruction cleanup recovery', () => { const { db } = await import('../../src/lib/server/db/index.js'); const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' }); + const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, + endpoint: { host: 'localhost', port: 28960 }, rawReply: '', + status: { players: 0, map: 'zm_transit', gametype: null, roster: [] } }); const name = 't6_zombie_town_1'; expect(await createServer({ ...input, game: 't6', mapSlug: 'town' })).toMatchObject({ ok: true, name }); expect(await createServer(input)).toMatchObject({ ok: true }); @@ -61,6 +71,7 @@ describe('destruction cleanup recovery', () => { db().prepare("INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, 'test.gsc')").run(beta, server); } const queued = destroyServer(name); + vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' }); await processDestructions(); expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); for (const table of ['beta_test_runs', 'beta_test_coverage', 'beta_file_changes']) { @@ -269,6 +280,12 @@ describe('T6 Zombies provisioning', () => { vi.mocked(rconStatusResult).mockImplementation(async (server) => ({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: '', status: { players: 0, map: server.startsWith('t6_') ? 'zm_transit' : 'zombie_cosmodrome', gametype: null, roster: [] } })); await processCreations(); + expect(getDeployment(t5.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' }); + expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'queued' }); + await processCreations(); + expect(getDeployment(t5.deploymentId)?.deployment).toMatchObject({ state: 'done' }); + expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' }); + await processCreations(); expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'done' }); const content = await readFile(join(fixture.t6, t6.name, 'configs/electric_cherry_map.cfg'), 'utf8'); expect(content).toContain('execgts zm_standard_town.cfg map zm_transit'); @@ -340,10 +357,13 @@ describe('server creation map selection', () => { closeDb(); interruptDeployments(); expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'queued' }); - vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }).mockResolvedValueOnce({ ok: false, reason: 'starting' }); await Promise.all([processCreations(), processCreations()]); - expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null }); + if (second.ok) expect(getDeployment(second.deploymentId)?.deployment).toMatchObject({ state: 'queued' }); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }); + await processCreations(); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); }); it('completes creation only when Komodo, RCON, and the selected map all pass', async () => { @@ -469,7 +489,8 @@ describe('server creation map selection', () => { expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'failed', error: 'Deploy: test failure' })); vi.mocked(komodoDeploy).mockRejectedValueOnce(new Error('connection lost')); await expect(createServer(input)).rejects.toThrow('connection lost'); - expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'failed', error: 'connection lost' })); + expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'awaiting-health', + stage: 'Verifying existing server after interrupted startup', finished_at: null })); const id = beginDeployment('interrupted-server', 'destroy'); interruptDeployments(); expect(listDeployments()).toContainEqual(expect.objectContaining({ id, state: 'interrupted', finished_at: expect.any(String) })); @@ -587,12 +608,12 @@ describe('persistent server keys', () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { getServerKey, listServerKeys } = await import('../../src/lib/server/fleet/server-keys.js'); const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, kind: 'not-found', message: 'gone' } as never); const name = 't5_zombie_cosmodrome_2'; expect(await createServer(input)).toMatchObject({ ok: true, name: serverName }); expect(await createServer(input)).toMatchObject({ ok: true, name }); expect(getServerKey(name)).toBe('test-key'); expect(await destroyServer(name)).toMatchObject({ ok: true }); + vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' }); await processDestructions(); expect(listServerKeys()).toContainEqual(expect.objectContaining({ server_id: name, deployed: 0 })); expect(JSON.stringify(listServerKeys())).not.toContain('test-key'); @@ -621,10 +642,10 @@ describe('persistent server keys', () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); - vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false } as never); expect(await createServer(input)).toMatchObject({ ok: true, name: serverName }); expect(await createServer(input)).toMatchObject({ ok: true, name: 't5_zombie_cosmodrome_2' }); await destroyServer(serverName); + vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' }); await processDestructions(); expect(await createServer({ ...input, plutoKey: 'wrong-key', expectedName: 't5_zombie_cosmodrome_3' })) .toMatchObject({ ok: false, kind: 'no-slot' }); diff --git a/tests/unit/creation-queue.test.ts b/tests/unit/creation-queue.test.ts new file mode 100644 index 0000000..548a498 --- /dev/null +++ b/tests/unit/creation-queue.test.ts @@ -0,0 +1,357 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ + komodoCreateDeployment: vi.fn(), + komodoDeploy: vi.fn(), + komodoDestroyDeployment: vi.fn(), + komodoDeleteDeployment: vi.fn(), + komodoInspectHealthy: vi.fn(), + komodoEnsureTag: vi.fn(), + komodoUpdateTagsOnResource: vi.fn() +})); +vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ + rconGetPlayerCount: vi.fn(), + rconStatusResult: vi.fn() +})); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); + +let fixture: Fixture; +const input = { gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'test-key' }; +function status(server: string, map = server.includes('_moon_') ? 'zombie_moon' : 'zombie_theater') { + return { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: '', + status: { players: 0, map, gametype: null, roster: [] } }; +} + +beforeEach(async () => { + vi.resetModules(); vi.resetAllMocks(); + fixture = await newFixture([]); + vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); + vi.stubEnv('KOMODO_API_KEY', 'test-key'); + vi.stubEnv('KOMODO_API_SECRET', 'test-secret'); + vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host'); + vi.stubEnv('RCON_PASSWORD', 'test-password'); + vi.stubEnv('SERVER_PORT_MIN', '28960'); + vi.stubEnv('SERVER_PORT_MAX', '28980'); + vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1'); + const komodo = await import('../../src/lib/server/integrations/komodo.js'); + vi.mocked(komodo.komodoCreateDeployment).mockResolvedValue({ ok: true }); + vi.mocked(komodo.komodoDeploy).mockResolvedValue({ ok: true }); + vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: true }); + vi.mocked(komodo.komodoDeleteDeployment).mockResolvedValue({ ok: true }); + vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name })); + vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true }); + const rcon = await import('../../src/lib/server/integrations/rcon.js'); + vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0); + vi.mocked(rcon.rconStatusResult).mockImplementation(async server => status(server)); + const { kick } = await import('../../src/lib/server/pipeline/worker.js'); + vi.mocked(kick).mockResolvedValue(undefined); +}); +afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); }); + +async function queueServers(count: number) { + const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); + return Array.from({ length: count }, () => { + const result = queueCreateServer(input); + if (!result.ok) throw new Error(result.message); + return result; + }); +} +async function states() { + const { db } = await import('../../src/lib/server/db/index.js'); + return (db().prepare("SELECT state FROM server_deployments WHERE action = 'create' ORDER BY rowid").all() as { state: string }[]).map(row => row.state); +} + +describe('shared creation queue', () => { + it('starts only one server per pass and holds its slot until a later health check', async () => { + const jobs = await queueServers(3); + const { processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + await processCreations(); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); + expect(komodoInspectHealthy).not.toHaveBeenCalled(); + expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']); + await processCreations(); + expect(komodoInspectHealthy).toHaveBeenCalledWith(jobs[0].name); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); + expect(await states()).toEqual(['done', 'awaiting-health', 'queued']); + expect(new Set(jobs.map(job => job.port)).size).toBe(3); + }); + + it('keeps later boots queued through container, RCON, and expected-map failures', async () => { + const jobs = await queueServers(2); + const { processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); + const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); + await processCreations(); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }); + await processCreations(); + expect(rconStatusResult).not.toHaveBeenCalled(); + expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for Komodo health: starting' }); + vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, reason: 'timeout', endpoint: null, detail: 'booting' }); + await processCreations(); + expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for RCON health: timeout' }); + vi.mocked(rconStatusResult).mockResolvedValueOnce(status(jobs[0].name, 'zombie_moon')); + await processCreations(); + expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: expect.stringContaining('expected zombie_theater') }); + vi.mocked(komodoInspectHealthy).mockRejectedValueOnce(new Error('Temporary connection error')); + await processCreations(); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); + expect(await states()).toEqual(['awaiting-health', 'queued']); + await processCreations(); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); + expect(await states()).toEqual(['done', 'awaiting-health']); + }); + + it('shares FIFO capacity across collections and direct callers, including equal timestamps', async () => { + const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); + const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); + const collections = ['theater', 'moon'].map(mapSlug => { + const blueprint = saveBlueprint({ name: mapSlug, entries: [{ game: 't5', mapSlug, count: 2 }] }); + const preview = previewCollection(blueprint.id); + return queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name, 'test-key'])) }); + }); + db().prepare("UPDATE server_deployments SET started_at = '2026-01-01T00:00:00.000Z'").run(); + const direct = await createServer(input); + if (!direct.ok) throw new Error(direct.message); + expect(await states()).toEqual(['awaiting-health', 'queued', 'queued', 'queued', 'queued']); + const second = getCollection(collections[1].collectionId!)!; + expect(second.collection).toMatchObject({ queued: 2, provisioning: 0, verifying: 0 }); + expect(second.queue).toMatchObject({ concurrency: 1, queued: 4, active: [{ server: 't5_zombie_theater_1' }] }); + expect(second.servers.map(server => server.queuePosition)).toEqual([2, 3]); + for (let pass = 0; pass < 5; pass++) await processCreations(); + expect(vi.mocked(komodoCreateDeployment).mock.calls.map(([spec]) => spec.name)).toEqual([ + 't5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1', 't5_zombie_moon_2', direct.name + ]); + expect(await states()).toEqual(['done', 'done', 'done', 'done', 'done']); + }); + + it('coalesces overlapping worker passes and direct calls while provisioning is pending', async () => { + await queueServers(3); + const { processCreations, createServer } = await import('../../src/lib/server/fleet/create.js'); + const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); + let release!: () => void; + let entered!: () => void; + const blocked = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; }); + const first = processCreations(); + await started; + const second = processCreations(); + const direct = createServer(input); + expect(await states()).toEqual(['running', 'queued', 'queued', 'queued']); + release(); + await Promise.all([first, second, direct]); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); + expect(await states()).toEqual(['awaiting-health', 'queued', 'queued', 'queued']); + }); + + it('honors a higher limit without refilling its pass budget after immediate failures', async () => { + vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2'); + await queueServers(5); + const { processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); + vi.mocked(komodoCreateDeployment).mockResolvedValue({ ok: false, kind: 'http-error', message: 'Host unavailable' }); + await processCreations(); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); + expect(await states()).toEqual(['failed', 'failed', 'queued', 'queued', 'queued']); + vi.mocked(komodoCreateDeployment).mockResolvedValue({ ok: true }); + await processCreations(); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(4); + expect(await states()).toEqual(['failed', 'failed', 'awaiting-health', 'awaiting-health', 'queued']); + }); + + it('does not provision a selected job that was cancelled during an earlier startup', async () => { + vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2'); + const jobs = await queueServers(3); + const { processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { finishDeployment } = await import('../../src/lib/server/fleet/deployments.js'); + const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); + let release!: () => void; + let entered!: () => void; + const blocked = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; }); + const pass = processCreations(); + await started; + finishDeployment(jobs[1].deploymentId, { ok: false, message: 'Cancelled before startup' }); + release(); + await pass; + expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); + expect(await states()).toEqual(['awaiting-health', 'failed', 'queued']); + }); + + it('rechecks capacity when an existing server resumes verification during provisioning', async () => { + vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2'); + const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { finishDeployment, resumeVerification } = await import('../../src/lib/server/fleet/deployments.js'); + const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); + const old = await createServer(input); + if (!old.ok) throw new Error(old.message); + finishDeployment(old.deploymentId, { ok: false, message: 'Needs verification' }); + await queueServers(2); + let release!: () => void; + let entered!: () => void; + const blocked = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; }); + const pass = processCreations(); + await started; + expect(resumeVerification(old.deploymentId)).toBe(true); + release(); + await pass; + expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); + expect(await states()).toEqual(['awaiting-health', 'awaiting-health', 'queued']); + }); + + it('counts already verifying servers against a higher concurrency limit', async () => { + vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2'); + const jobs = await queueServers(4); + const { processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + await processCreations(); + vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === jobs[0].name + ? { ok: false, reason: 'still booting' } : { ok: true, state: 'running' }); + await processCreations(); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(3); + expect(await states()).toEqual(['awaiting-health', 'done', 'awaiting-health', 'queued']); + }); + + it('retains durable reservations and the startup gate across an application restart', async () => { + const jobs = await queueServers(3); + const { processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const { komodoCreateDeployment, komodoDeploy, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + await processCreations(); + db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(jobs[0].deploymentId); + closeDb(); + vi.resetModules(); + const { interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); + const { processCreations: resumed, nextAvailablePort } = await import('../../src/lib/server/fleet/create.js'); + interruptDeployments(); + expect(await states()).toEqual(['interrupted', 'queued', 'queued']); + expect(nextAvailablePort()).toBe(28963); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' }); + await resumed(); + expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); + expect(komodoDeploy).toHaveBeenCalledTimes(1); + await resumed(); + expect(await states()).toEqual(['done', 'awaiting-health', 'queued']); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); + }); + + it('keeps an ambiguous startup response in verification without replaying its deployment', async () => { + await queueServers(2); + const { processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { komodoDeploy, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + vi.mocked(komodoDeploy).mockRejectedValueOnce(new Error('Response lost after starting')); + const log = vi.spyOn(console, 'error').mockImplementation(() => {}); + await processCreations(); + expect(await states()).toEqual(['awaiting-health', 'queued']); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' }); + await processCreations(); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); + expect(komodoDeploy).toHaveBeenCalledTimes(1); + expect(log).toHaveBeenCalled(); + await processCreations(); + expect(await states()).toEqual(['done', 'awaiting-health']); + }); + + it.each([ + { ok: false as const, kind: 'unreachable' as const, message: 'Request timed out after Komodo accepted it' }, + { ok: false as const, kind: 'http-error' as const, httpStatus: 502, message: 'Gateway lost the startup response' } + ])('holds the slot and files for an uncertain structured response: $kind', async failure => { + const jobs = await queueServers(2); + const { processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + const { komodoDeploy, komodoDeleteDeployment, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + vi.mocked(komodoDeploy).mockResolvedValueOnce(failure); + await processCreations(); + expect(await states()).toEqual(['awaiting-health', 'queued']); + expect(getServer(jobs[0].name)?.game_port).toBe(jobs[0].port); + expect(existsSync(join(fixture.t5, jobs[0].name, 'configs/electric_cherry_map.cfg'))).toBe(true); + expect(komodoDeleteDeployment).not.toHaveBeenCalled(); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' }); + await processCreations(); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); + expect(komodoDeploy).toHaveBeenCalledTimes(1); + await processCreations(); + expect(await states()).toEqual(['done', 'awaiting-health']); + }); + + it('releases capacity on a confirmed deployment rejection', async () => { + const jobs = await queueServers(2); + const { processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + const { komodoDeploy, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); + vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', httpStatus: 403, message: 'Access denied' }); + await processCreations(); + expect(await states()).toEqual(['failed', 'queued']); + expect(getServer(jobs[0].name)).toBeUndefined(); + expect(existsSync(join(fixture.t5, jobs[0].name))).toBe(false); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); + await processCreations(); + expect(await states()).toEqual(['failed', 'awaiting-health']); + }); + + it('retains an interrupted startup slot while a newer destruction is still waiting', async () => { + const jobs = await queueServers(2); + const { processCreations, destroyServer } = await import('../../src/lib/server/fleet/create.js'); + const { interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + await processCreations(); + expect(destroyServer(jobs[0].name)).toMatchObject({ ok: true }); + db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(jobs[0].deploymentId); + interruptDeployments(); + expect(await states()).toEqual(['interrupted', 'queued']); + await processCreations(); + expect(await states()).toEqual(['awaiting-health', 'queued']); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); + expect(komodoInspectHealthy).not.toHaveBeenCalled(); + }); + + it('releases a startup slot after its server is removed', async () => { + const jobs = await queueServers(2); + const { processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + await processCreations(); + db().prepare('DELETE FROM servers WHERE name = ?').run(jobs[0].name); + await processCreations(); + expect(await states()).toEqual(['failed', 'awaiting-health']); + }); + + it('never verifies a replacement registered under the same server name', async () => { + const jobs = await queueServers(2); + const { processCreations } = await import('../../src/lib/server/fleet/create.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + await processCreations(); + db().prepare('UPDATE servers SET game_port = ? WHERE name = ?').run(28980, jobs[0].name); + await processCreations(); + expect(komodoInspectHealthy).not.toHaveBeenCalledWith(jobs[0].name); + expect(await states()).toEqual(['failed', 'awaiting-health']); + }); +}); + +describe('creation concurrency configuration', () => { + it('defaults to one startup', async () => { + vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', undefined); + const { config } = await import('../../src/lib/server/config.js'); + expect(config.createConcurrency).toBe(1); + }); + + it.each(['0', '-1', '1.5', 'invalid', 'Infinity', '9007199254740992'])('rejects invalid limit %s', async value => { + vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', value); + await expect(import('../../src/lib/server/config.js')).rejects.toThrow(/DEPLOYMENT_CREATE_CONCURRENCY/); + vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1'); + }); +}); diff --git a/tests/unit/deployment-collections.test.ts b/tests/unit/deployment-collections.test.ts index aefcb2f..aa579b8 100644 --- a/tests/unit/deployment-collections.test.ts +++ b/tests/unit/deployment-collections.test.ts @@ -36,6 +36,7 @@ beforeEach(async () => { vi.stubEnv('RCON_PASSWORD', 'private-rcon-password'); vi.stubEnv('SERVER_PORT_MIN', '28960'); vi.stubEnv('SERVER_PORT_MAX', '28980'); + vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1'); }); afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); @@ -60,6 +61,7 @@ describe('multi-server blueprints', () => { const stored = db().prepare('SELECT recipe FROM server_blueprints WHERE id = ?').get(blueprint.id) as { recipe: string }; expect(JSON.parse(stored.recipe)).toEqual({ format: 3, entries }); expect(preview.ready).toBe(true); + expect(preview.launchConcurrency).toBe(1); expect(preview.servers.map(server => server.name)).toEqual(['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1', 't6_zombie_town_1', 't6_zombie_town_2']); expect(preview.servers.map(server => server.port)).toEqual([28960, 28961, 28962, 28963, 28964]); expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]); @@ -112,7 +114,7 @@ describe('multi-server blueprints', () => { expect(preview.entries[0].recipe.files).toContainEqual(expect.objectContaining({ path: 'telemetry.gsc', versionId: current.id })); expect(preview.entries[2].recipe.mapSlug).toBe('town'); const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); - await processCreations(); + for (let pass = 0; pass <= preview.servers.length; pass++) await processCreations(); expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, done: 5 }); for (const name of ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1']) { expect(await readFile(join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/telemetry.gsc'), 'utf8')).toBe('current telemetry'); @@ -221,9 +223,14 @@ describe('atomic collection launches', () => { const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); expect(komodoCreateDeployment).not.toHaveBeenCalled(); - expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, active: 5, done: 0, failed: 0 }); + expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, active: 5, done: 0, failed: 0, queued: 5, provisioning: 0, verifying: 0 }); + expect(getCollection(queued.collectionId!)?.servers.map(server => server.queuePosition)).toEqual([1, 2, 3, 4, 5]); closeDb(); await processCreations(); + expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ queued: 4, provisioning: 0, verifying: 1, done: 0 }); + expect(getCollection(queued.collectionId!)?.queue).toMatchObject({ concurrency: 1, queued: 4, active: [{ server: preview.servers[0].name, state: 'awaiting-health' }] }); + expect(getCollection(queued.collectionId!)?.servers.map(server => server.queuePosition)).toEqual([null, 1, 2, 3, 4]); + for (let pass = 0; pass < preview.servers.length; pass++) await processCreations(); const detail = getCollection(queued.collectionId!)!; expect(detail.collection).toMatchObject({ total: 5, active: 0, done: 5, failed: 0 }); expect(listCollections()).toHaveLength(1); @@ -308,7 +315,7 @@ describe('atomic collection launches', () => { const newest = uploadNewVersion(script.id, Buffer.from('promoted while queued')); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newest.id, script.id); uploadNewVersion(script.id, Buffer.from('unpromoted draft')); - await processCreations(); + for (let pass = 0; pass <= preview.servers.length; pass++) await processCreations(); expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ name: 'Friday Zombies', total: 5, done: 5 }); for (const name of ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1']) { expect(await readFile(join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('promoted while queued'); @@ -323,6 +330,9 @@ describe('atomic collection launches', () => { vi.mocked(komodoCreateDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'Host rejected this container' }); const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); await processCreations(); + expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ queued: 4, failed: 1, done: 0 }); + expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); + for (let pass = 0; pass < preview.servers.length; pass++) await processCreations(); const detail = getCollection(queued.collectionId!)!; expect(detail.collection).toMatchObject({ total: 5, active: 0, done: 4, failed: 1 }); expect(detail.servers[0]).toMatchObject({ state: 'failed', error: expect.stringContaining('Host rejected') }); diff --git a/tests/unit/komodo-write.test.ts b/tests/unit/komodo-write.test.ts new file mode 100644 index 0000000..036dac5 --- /dev/null +++ b/tests/unit/komodo-write.test.ts @@ -0,0 +1,29 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +beforeEach(() => { + vi.resetModules(); + vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); + vi.stubEnv('KOMODO_API_KEY', 'test-key'); + vi.stubEnv('KOMODO_API_SECRET', 'test-secret'); +}); +afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); }); + +describe('Komodo startup response classification', () => { + it.each([403, 502])('preserves HTTP %i so callers can distinguish rejection from uncertain startup', async httpStatus => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('Deployment response failed', { status: httpStatus }))); + const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); + expect(await komodoDeploy('t5_zombie_theater_1')).toMatchObject({ ok: false, kind: 'http-error', httpStatus }); + }); + + it('identifies an explicit success=false response as a confirmed HTTP 200 failure', async () => { + vi.stubGlobal('fetch', vi.fn(async () => Response.json({ success: false, message: 'Image rejected' }))); + const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); + expect(await komodoDeploy('t5_zombie_theater_1')).toMatchObject({ ok: false, kind: 'http-error', httpStatus: 200, message: 'Image rejected' }); + }); + + it('returns a structured unreachable result when the startup response times out', async () => { + vi.stubGlobal('fetch', vi.fn(async () => { throw new DOMException('Request aborted', 'AbortError'); })); + const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); + expect(await komodoDeploy('t5_zombie_theater_1')).toMatchObject({ ok: false, kind: 'unreachable', message: expect.stringContaining('timed out') }); + }); +}); diff --git a/tests/unit/launch-recipes.test.ts b/tests/unit/launch-recipes.test.ts index 0f9bb89..96a0500 100644 --- a/tests/unit/launch-recipes.test.ts +++ b/tests/unit/launch-recipes.test.ts @@ -100,6 +100,8 @@ describe('launch recipes and previews', () => { const recordedConfig = recipeForDeployment(queued.deploymentId)?.recipe.files.find(file => file.path === 'dedicated_sp.cfg'); expect(Buffer.from(recordedConfig!.content, 'base64').toString()).toBe(deployedConfig); expect(getDeployment(queued.deploymentId)?.recipe?.fingerprint).not.toBe(preview.recipe.fingerprint); + expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' }); + await processCreations(); expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'done' }); expect(JSON.stringify(getDeployment(queued.deploymentId))).not.toContain('private-'); });