From a611850649828fbee43b360eb4f45892132c053a Mon Sep 17 00:00:00 2001 From: josh Date: Thu, 17 Sep 2026 16:57:08 -0400 Subject: [PATCH] new destruction flow --- README.md | 2 + src/lib/server/db/schema.sql | 8 ++ src/lib/server/fleet/create.ts | 66 ++++++++--- src/lib/server/fleet/deployments.ts | 20 ++++ src/lib/server/pipeline/worker.ts | 3 + .../ui/components/DestroyServerModal.svelte | 59 ++++++++++ src/routes/api/deployments/[id]/+server.ts | 3 + src/routes/api/servers/[name]/+server.ts | 4 +- src/routes/deployments/+page.svelte | 24 ++-- src/routes/deployments/[id]/+page.server.ts | 11 +- src/routes/deployments/[id]/+page.svelte | 105 +++++++++++++++++- src/routes/servers/[name]/+page.svelte | 69 +----------- tests/unit/create-server.test.ts | 71 +++++++++++- 13 files changed, 349 insertions(+), 96 deletions(-) create mode 100644 src/lib/ui/components/DestroyServerModal.svelte diff --git a/README.md b/README.md index ce50f70..c49649b 100644 --- a/README.md +++ b/README.md @@ -204,3 +204,5 @@ until the parity check passes. Campaigns (`/campaigns`) contains script/config rollouts, drift reconciliation, restart progress, file diffs, and rollbacks. Existing rollout records are preserved in the original database tables. The campaign API lives at `/api/campaigns`; legacy rollout detail API endpoints remain available for compatibility. Deployments (`/deployments`, `/api/deployments`) now tracks server creations and destructions with live stages and persistent history, including failures, warnings, and interrupted operations. History starts with operations performed after this update and survives server removal. Creation completion means the server was provisioned and Komodo accepted the container start request; runtime health remains on the Fleet page. + +Each server deployment opens a detail page with its flow and timestamped activity. Destroy uses an in-app confirmation modal and immediately returns a queued deployment (`DELETE /api/servers/:name` returns HTTP 202 with `deploymentId`). The worker waits for a fresh RCON player count of zero before teardown; occupied or unreachable servers remain queued and are checked each worker tick. Waiting requests survive application restarts, while interrupted teardown is marked for inspection. Completed destruction stays on the deployment detail page. diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index 88c0b9c..2f81a97 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -268,3 +268,11 @@ CREATE TABLE IF NOT EXISTS server_deployments ( error TEXT, warnings TEXT NOT NULL DEFAULT '[]' ); + +CREATE TABLE IF NOT EXISTS server_deployment_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + deployment_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE, + created_at TEXT NOT NULL, + message TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_server_deployment_events ON server_deployment_events(deployment_id, id); diff --git a/src/lib/server/fleet/create.ts b/src/lib/server/fleet/create.ts index b7afb9e..dc77de3 100644 --- a/src/lib/server/fleet/create.ts +++ b/src/lib/server/fleet/create.ts @@ -1,3 +1,4 @@ +import { rconGetPlayerCount } from '../integrations/rcon.js'; import { beginDeployment, deploymentStage, finishDeployment } from './deployments.js'; import { mkdir, chmod, rm, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; @@ -334,19 +335,56 @@ async function applyDeploymentTags(deployment: string, gameTagName: string, mapT } const destroying = new Set(); -export async function destroyServer(name: string): Promise { - if (creating || destroying.has(name)) return { ok: false, kind: 'komodo', message: 'A server operation is already in progress. Retry shortly.', warnings: [] }; + +// Persist the request before returning so navigation and client disconnects cannot lose it. +export function destroyServer(name: string): DestroyServerResult & { deploymentId?: string } { + const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined; + if (existing) return { ok: true, warnings: [], deploymentId: existing.id }; if (!getServer(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] }; - const operation = beginDeployment(name, 'destroy'); - destroying.add(name); - try { - const result = await destroyServerUnlocked(name, operation); - finishDeployment(operation, result); - return result; - } catch (err) { - finishDeployment(operation, { ok: false, message: (err as Error).message }); - throw err; - } finally { destroying.delete(name); } + const operation = db().transaction(() => { + const id = beginDeployment(name, 'destroy'); + db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); + deploymentStage(id, 'Waiting to verify the server is empty'); + return id; + })(); + return { ok: true, warnings: [], deploymentId: operation }; +} + +let destructionTick: Promise | null = null; +export function processDestructions(): Promise { + if (destructionTick) return destructionTick; + destructionTick = runDestructions().finally(() => { destructionTick = null; }); + return destructionTick; +} + +async function runDestructions(): Promise { + const queued = db().prepare("SELECT id, server FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string }[]; + for (const operation of queued) { + if (creating || destroying.has(operation.server)) continue; + destroying.add(operation.server); + try { + if (!getServer(operation.server)) { + finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' }); + continue; + } + // A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness. + const players = await rconGetPlayerCount(operation.server); + if (players !== 0) { + deploymentStage(operation.id, players === null + ? 'Waiting for player count: server could not be reached' + : 'Waiting for server to empty: ' + players + ' player(s) online'); + continue; + } + deploymentStage(operation.id, 'Server is empty (0 players verified)'); + db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(operation.id); + const result = await destroyServerUnlocked(operation.server, operation.id); + finishDeployment(operation.id, result); + } catch (err) { + const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string }; + if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message); + else finishDeployment(operation.id, { ok: false, message: (err as Error).message }); + } finally { destroying.delete(operation.server); } + } } async function destroyServerUnlocked(name: string, operation: string): Promise { @@ -362,7 +400,7 @@ async function destroyServerUnlocked(name: string, operation: string): Promise { } async function runTick(): Promise { + try { await processDestructions(); } + catch (err) { console.error('[worker] destruction error:', err); } try { await sweep(); } catch (err) { diff --git a/src/lib/ui/components/DestroyServerModal.svelte b/src/lib/ui/components/DestroyServerModal.svelte new file mode 100644 index 0000000..3002489 --- /dev/null +++ b/src/lib/ui/components/DestroyServerModal.svelte @@ -0,0 +1,59 @@ + + + open = false} + oncancel={(event) => { if (submitting) event.preventDefault(); }}> +
{ event.preventDefault(); void destroy(); }}> +

Destroy container?

+

{name}

+

This stops the container, deletes its Komodo deployment, and removes its server files and campaign details. Deployment history and the saved server key are retained.

+
+ {#if players !== null && players > 0}{players} player{players === 1 ? '' : 's'} currently online.{/if} + Destruction waits until a live check confirms the server is empty. You can follow its progress on the deployment page. +
+

Once destruction begins, it cannot be undone.

+ {#if error}{/if} +
+ + +
+
+
+ + diff --git a/src/routes/api/deployments/[id]/+server.ts b/src/routes/api/deployments/[id]/+server.ts index 7aed1fc..bf2895b 100644 --- a/src/routes/api/deployments/[id]/+server.ts +++ b/src/routes/api/deployments/[id]/+server.ts @@ -1,7 +1,10 @@ import { json, error } from '@sveltejs/kit'; import { db } from '$lib/server/db/index.js'; +import { getDeployment } from '$lib/server/fleet/deployments.js'; export async function GET({ params }: { params: { id: string } }) { + const lifecycle = getDeployment(params.id); + if (lifecycle) return json(lifecycle); const dep = db() .prepare('SELECT * FROM deployments WHERE id = ?') .get(params.id) as Record | undefined; diff --git a/src/routes/api/servers/[name]/+server.ts b/src/routes/api/servers/[name]/+server.ts index 89d08ea..6afc3ff 100644 --- a/src/routes/api/servers/[name]/+server.ts +++ b/src/routes/api/servers/[name]/+server.ts @@ -1,3 +1,4 @@ +import { kick } from '$lib/server/pipeline/worker.js'; import { json } from '@sveltejs/kit'; import { destroyServer } from '$lib/server/fleet/create.js'; @@ -11,5 +12,6 @@ export async function DELETE({ params }: { params: { name: string } }) { const status = result.kind === 'not-found' ? 404 : 500; return json({ ok: false, error: result.message, kind: result.kind, warnings: result.warnings }, { status }); } - return json({ ok: true, warnings: result.warnings }); + void kick().catch((err) => console.error('[destroy] worker kick failed:', err)); + return json({ ok: true, deploymentId: result.deploymentId }, { status: 202 }); } diff --git a/src/routes/deployments/+page.svelte b/src/routes/deployments/+page.svelte index 2a67e00..16eebf3 100644 --- a/src/routes/deployments/+page.svelte +++ b/src/routes/deployments/+page.svelte @@ -8,7 +8,7 @@ interface Deployment { id: string; server: string; action: 'create' | 'destroy'; - state: 'running' | 'done' | 'failed' | 'warning' | 'interrupted'; + state: 'waiting' | 'running' | 'done' | 'failed' | 'warning' | 'interrupted'; stage: string; started_at: string; finished_at: string | null; triggered_by: string; error: string | null; warnings: string; server_exists: number; } @@ -19,8 +19,8 @@ let action = $state('all'); let search = $state(''); let refreshing = false; - let active = $derived(deployments.filter(d => d.state === 'running')); - let history = $derived(deployments.filter(d => d.state !== 'running' && + let active = $derived(deployments.filter(d => (d.state === 'running' || d.state === 'waiting'))); + let history = $derived(deployments.filter(d => d.state !== 'running' && d.state !== 'waiting' && (action === 'all' || d.action === action) && d.server.toLowerCase().includes(search.toLowerCase()))); async function load() { if (refreshing) return; @@ -41,11 +41,11 @@ function fmt(value: string) { return new Date(value).toLocaleString(); } function warnings(d: Deployment): string[] { try { return JSON.parse(d.warnings); } catch { return []; } } function status(d: Deployment) { - return { running: 'In flight', done: 'Completed', failed: 'Failed', warning: 'Completed with warnings', interrupted: 'Interrupted' }[d.state]; + return { waiting: 'Waiting for empty', running: 'In flight', done: 'Completed', failed: 'Failed', warning: 'Completed with warnings', interrupted: 'Interrupted' }[d.state]; } -Server deployments ? Electric Cherry +Server deployments | Electric Cherry
@@ -55,7 +55,7 @@ {#if error}{/if}

In flight {active.length}

- {#if loading}

Loading deployments?

+ {#if loading}

Loading deployments...

{:else if active.length === 0} {:else}
{#each active as d (d.id)}{@render row(d)}{/each}
{/if}
@@ -71,10 +71,10 @@
{#snippet row(d: Deployment)} - + {/snippet} diff --git a/src/routes/servers/[name]/+page.svelte b/src/routes/servers/[name]/+page.svelte index d395bde..caf9aff 100644 --- a/src/routes/servers/[name]/+page.svelte +++ b/src/routes/servers/[name]/+page.svelte @@ -2,7 +2,7 @@ import RoundTelemetry from '$ui/components/RoundTelemetry.svelte'; import { onMount, onDestroy } from 'svelte'; import { page } from '$app/stores'; - import { goto } from '$app/navigation'; + import DestroyServerModal from '$ui/components/DestroyServerModal.svelte'; import { statusByServer, uiNow } from '$ui/stores/servers.js'; import type { RosterPlayer, Health, RconEndpoint, RconFailure } from '$ui/stores/servers.js'; import { elapsedSince } from '$ui/duration.js'; @@ -90,9 +90,7 @@ let diagnosing = $state(false); let rconProbe = $state(null); let probingRcon = $state(false); - let destroying = $state(false); - let destroyError = $state(null); - let destroyWarnings = $state([]); + let destroyOpen = $state(false); let activeTab = $state<'roster' | 'logs' | 'inventory' | 'diagnostics' | 'history'>('roster'); async function loadServer(): Promise { @@ -207,35 +205,6 @@ rconProbe = { ok: false, reason: 'send-error', detail: (err as Error).message }; } finally { probingRcon = false; } } - async function destroyThisServer(): Promise { - if (destroying || !name) return; - const confirmed = confirm( - `Destroy ${name}?\n\nThis stops the container, deletes the Komodo deployment, ` + - `removes the host directory, and removes this server?s campaign details. Server deployment history is retained. Cannot be undone.` - ); - if (!confirmed) return; - destroying = true; destroyError = null; destroyWarnings = []; - try { - const res = await fetch(`/api/servers/${encodeURIComponent(name)}`, { method: 'DELETE' }); - const body = (await res.json()) as { ok: boolean; error?: string; warnings?: string[] }; - if (!body.ok) { - destroyError = body.error ?? 'destroy failed'; - destroyWarnings = body.warnings ?? []; - destroying = false; - return; - } - if (body.warnings && body.warnings.length > 0) { - destroyWarnings = body.warnings; - setTimeout(() => goto('/'), 1500); - return; - } - await goto('/'); - } catch (err) { - destroyError = (err as Error).message; - destroying = false; - } - } - // Tab config — Diagnostics gets a warning badge when there's an issue. let tabs = $derived([ { id: 'roster', label: 'Roster', badge: effectiveRoster?.length ?? null }, @@ -246,6 +215,8 @@ ]); + + {#if server}
@@ -255,7 +226,7 @@ {/snippet} {#snippet actions()} - {/snippet} @@ -307,14 +278,6 @@ - {#if destroyError} - - {/if} - {#if destroyWarnings.length > 0} -
    - {#each destroyWarnings as w}
  • {w}
  • {/each} -
- {/if}
@@ -604,27 +567,7 @@ color: var(--text-tertiary); } - .destroy-error { - margin-top: var(--space-4); - padding: var(--space-3) var(--space-4); - background: rgba(255, 86, 112, 0.08); - border: 1px solid rgba(255, 86, 112, 0.28); - border-radius: var(--radius-md); - color: var(--state-danger); - font-size: var(--text-sm); - } - .destroy-warn { - margin: var(--space-3) 0 0; - padding: var(--space-3) var(--space-5); - background: rgba(255, 176, 32, 0.08); - border: 1px solid rgba(255, 176, 32, 0.28); - border-radius: var(--radius-md); - color: var(--state-warning); - font-size: var(--text-sm); - list-style: disc inside; - } - - /* Tabs row */ +/* Tabs row */ .tabs-row { display: flex; justify-content: center; diff --git a/tests/unit/create-server.test.ts b/tests/unit/create-server.test.ts index f12f448..f3bc9d4 100644 --- a/tests/unit/create-server.test.ts +++ b/tests/unit/create-server.test.ts @@ -13,6 +13,7 @@ vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })), komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true })) })); +vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(async () => 0) })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); let fixture: Fixture; @@ -53,8 +54,69 @@ async function promotedConfig() { } describe('server creation map selection', () => { + it('returns a deployment URL identifier immediately and serves its lifecycle detail', async () => { + const { createServer } = await import('../../src/lib/server/fleet/create.js'); + const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js'); + const { GET } = await import('../../src/routes/api/deployments/[id]/+server.js'); + const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); + await createServer(input); + const response = await DELETE({ params: { name: serverName } }); + expect(response.status).toBe(202); + const body = await response.json(); + expect(body).toMatchObject({ ok: true, deploymentId: expect.any(String) }); + expect(komodoDestroyDeployment).not.toHaveBeenCalled(); + const detail = await (await GET({ params: { id: body.deploymentId } })).json(); + expect(detail.deployment).toMatchObject({ server: serverName, action: 'destroy', state: 'waiting' }); + expect(detail.events).toHaveLength(2); + const missing = await DELETE({ params: { name: 'missing-server' } }); + expect(missing.status).toBe(404); + }); + + it('queues destruction, waits for live zero players, and preserves an ordered timeline', async () => { + const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); + const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); + const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); + const { closeDb } = await import('../../src/lib/server/db/index.js'); + await createServer(input); + const queued = destroyServer(serverName); + expect(queued).toMatchObject({ ok: true, deploymentId: expect.any(String) }); + expect(destroyServer(serverName).deploymentId).toBe(queued.deploymentId); + expect(komodoDestroyDeployment).not.toHaveBeenCalled(); + vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(3).mockResolvedValueOnce(null).mockRejectedValueOnce(new Error('RCON unavailable')); + for (let i = 0; i < 3; i++) { + await processDestructions(); + expect(komodoDestroyDeployment).not.toHaveBeenCalled(); + expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting' }); + } + closeDb(); + interruptDeployments(); + expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting' }); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }); + await Promise.all([processDestructions(), processDestructions()]); + expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1); + const detail = getDeployment(queued.deploymentId!); + expect(detail?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); + const messages = (detail?.events as { message: string }[]).map(e => e.message); + expect(messages).toEqual(expect.arrayContaining(['Server destruction requested', 'Waiting for server to empty: 3 player(s) online', 'Server is empty (0 players verified)', 'Stopping and removing container', 'Completed'])); + expect(messages.indexOf('Server is empty (0 players verified)')).toBeLessThan(messages.indexOf('Stopping and removing container')); + }); + + it('preserves files and registry when Komodo rejects destruction and exposes failure details', async () => { + const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); + const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); + const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); + await createServer(input); + vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'denied' }); + const queued = destroyServer(serverName); + await processDestructions(); + expect(komodoDeleteDeployment).not.toHaveBeenCalled(); + expect(existsSync(join(fixture.t5, serverName))).toBe(true); + expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'failed', server_exists: 1, error: 'DestroyDeployment failed: denied' }); + }); + it('records in-flight creation and retains both operations after destruction', async () => { - const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); + const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { listDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { komodoDeploy, komodoInspectHealthy, komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoDeploy).mockImplementationOnce(async () => { @@ -68,6 +130,7 @@ describe('server creation map selection', () => { return { ok: true }; }); expect((await destroyServer(serverName)).ok).toBe(true); + await processDestructions(); expect(listDeployments()).toEqual(expect.arrayContaining([ expect.objectContaining({ server: serverName, action: 'create', state: 'done', server_exists: 0 }), expect.objectContaining({ server: serverName, action: 'destroy', state: 'done', server_exists: 0 }) @@ -198,7 +261,7 @@ describe('server creation map selection', () => { describe('persistent server keys', () => { it('saves a key, retains it through destruction, and reuses it for the exact lobby', async () => { - const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); + const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { getServerKey, listServerKeys } = await import('../../src/lib/server/fleet/server-keys.js'); const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, kind: 'not-found', message: 'gone' } as never); @@ -207,6 +270,7 @@ describe('persistent server keys', () => { expect(await createServer(input)).toMatchObject({ ok: true, name }); expect(getServerKey(name)).toBe('test-key'); expect(await destroyServer(name)).toMatchObject({ ok: true }); + await processDestructions(); expect(listServerKeys()).toContainEqual(expect.objectContaining({ server_id: name, deployed: 0 })); expect(JSON.stringify(listServerKeys())).not.toContain('test-key'); expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', expectedName: name })).toMatchObject({ ok: true, name }); @@ -231,13 +295,14 @@ describe('persistent server keys', () => { }); it('fills the lowest gap and rejects stale previews before saving a key or deploying', async () => { - const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); + const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false } as never); expect(await createServer(input)).toMatchObject({ ok: true, name: serverName }); expect(await createServer(input)).toMatchObject({ ok: true, name: 't5_zombie_cosmodrome_2' }); await destroyServer(serverName); + await processDestructions(); expect(await createServer({ ...input, plutoKey: 'wrong-key', expectedName: 't5_zombie_cosmodrome_3' })) .toMatchObject({ ok: false, kind: 'no-slot' }); expect(getServerKey(serverName)).toBe('test-key');