new destruction flow

src/lib/server/db/schema.sql

@@ -268,3 +268,11 @@ CREATE TABLE IF NOT EXISTS server_deployments (
     error TEXT,
     warnings TEXT NOT NULL DEFAULT '[]'
 );
+
+CREATE TABLE IF NOT EXISTS server_deployment_events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    deployment_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE,
+    created_at TEXT NOT NULL,
+    message TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_server_deployment_events ON server_deployment_events(deployment_id, id);

src/lib/server/fleet/create.ts

@@ -1,3 +1,4 @@
+import { rconGetPlayerCount } from '../integrations/rcon.js';
 import { beginDeployment, deploymentStage, finishDeployment } from './deployments.js';
 import { mkdir, chmod, rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
@@ -334,19 +335,56 @@ async function applyDeploymentTags(deployment: string, gameTagName: string, mapT
 }
 
 const destroying = new Set<string>();
-export async function destroyServer(name: string): Promise<DestroyServerResult> {
-    if (creating || destroying.has(name)) return { ok: false, kind: 'komodo', message: 'A server operation is already in progress. Retry shortly.', warnings: [] };
+
+// Persist the request before returning so navigation and client disconnects cannot lose it.
+export function destroyServer(name: string): DestroyServerResult & { deploymentId?: string } {
+    const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
+    if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
     if (!getServer(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] };
-    const operation = beginDeployment(name, 'destroy');
-    destroying.add(name);
-    try {
-        const result = await destroyServerUnlocked(name, operation);
-        finishDeployment(operation, result);
-        return result;
-    } catch (err) {
-        finishDeployment(operation, { ok: false, message: (err as Error).message });
-        throw err;
-    } finally { destroying.delete(name); }
+    const operation = db().transaction(() => {
+        const id = beginDeployment(name, 'destroy');
+        db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
+        deploymentStage(id, 'Waiting to verify the server is empty');
+        return id;
+    })();
+    return { ok: true, warnings: [], deploymentId: operation };
+}
+
+let destructionTick: Promise<void> | null = null;
+export function processDestructions(): Promise<void> {
+    if (destructionTick) return destructionTick;
+    destructionTick = runDestructions().finally(() => { destructionTick = null; });
+    return destructionTick;
+}
+
+async function runDestructions(): Promise<void> {
+    const queued = db().prepare("SELECT id, server FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string }[];
+    for (const operation of queued) {
+        if (creating || destroying.has(operation.server)) continue;
+        destroying.add(operation.server);
+        try {
+            if (!getServer(operation.server)) {
+                finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' });
+                continue;
+            }
+            // A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
+            const players = await rconGetPlayerCount(operation.server);
+            if (players !== 0) {
+                deploymentStage(operation.id, players === null
+                    ? 'Waiting for player count: server could not be reached'
+                    : 'Waiting for server to empty: ' + players + ' player(s) online');
+                continue;
+            }
+            deploymentStage(operation.id, 'Server is empty (0 players verified)');
+            db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(operation.id);
+            const result = await destroyServerUnlocked(operation.server, operation.id);
+            finishDeployment(operation.id, result);
+        } catch (err) {
+            const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string };
+            if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message);
+            else finishDeployment(operation.id, { ok: false, message: (err as Error).message });
+        } finally { destroying.delete(operation.server); }
+    }
 }
 
 async function destroyServerUnlocked(name: string, operation: string): Promise<DestroyServerResult> {
@@ -362,7 +400,7 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
     deploymentStage(operation, 'Stopping and removing container');
     const destroyRes = await komodoDestroyDeployment(name);
     if (!destroyRes.ok) {
-        warnings.push(`DestroyDeployment failed: ${destroyRes.message}`);
+        return { ok: false, kind: 'komodo', message: 'DestroyDeployment failed: ' + destroyRes.message, warnings };
     }
 
     // 2: wait for the container to actually disappear. Komodo refuses to
@@ -380,7 +418,7 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
     deploymentStage(operation, 'Deleting Komodo deployment');
     const deleteRes = await komodoDeleteDeployment(name);
     if (!deleteRes.ok) {
-        warnings.push(`DeleteDeployment failed: ${deleteRes.message}`);
+        return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings };
     }
 
     // 3: host directory. Fall back to derived path if host_dir wasn't set on

src/lib/server/fleet/deployments.ts

@@ -5,17 +5,35 @@ export function beginDeployment(server: string, action: 'create' | 'destroy', tr
     const id = randomUUID();
     db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
         VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
+    recordDeploymentEvent(id, action === 'create' ? 'Server creation requested' : 'Server destruction requested');
     return id;
 }
 
 export function deploymentStage(id: string, stage: string): void {
+    const previous = db().prepare('SELECT stage FROM server_deployments WHERE id = ?').get(id) as { stage: string } | undefined;
     db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id);
+    if (previous && previous.stage !== stage) recordDeploymentEvent(id, stage);
+}
+
+export function recordDeploymentEvent(id: string, message: string): void {
+    db().prepare('INSERT INTO server_deployment_events (deployment_id, created_at, message) VALUES (?, ?, ?)')
+        .run(id, new Date().toISOString(), message);
+}
+
+export function getDeployment(id: string) {
+    const deployment = db().prepare(`SELECT d.*, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
+        FROM server_deployments d WHERE d.id = ?`).get(id);
+    if (!deployment) return null;
+    const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id);
+    return { deployment, events };
 }
 
 export function finishDeployment(id: string, result: { ok: boolean; message?: string; warnings?: string[] }): void {
     const state = !result.ok ? 'failed' : result.warnings?.length ? 'warning' : 'done';
     db().prepare(`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`)
         .run(state, new Date().toISOString(), result.message ?? null, JSON.stringify(result.warnings ?? []), id);
+    for (const warning of result.warnings ?? []) recordDeploymentEvent(id, warning);
+    recordDeploymentEvent(id, result.ok ? (state === 'warning' ? 'Completed with warnings' : 'Completed') : `Failed: ${result.message ?? 'Unknown error'}`);
 }
 
 export function listDeployments() {
@@ -24,7 +42,9 @@ export function listDeployments() {
 }
 
 export function interruptDeployments(): void {
+    const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running'").all() as { id: string }[];
     db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?,
         error = 'The application restarted before this operation finished. Check the server before retrying.'
         WHERE state = 'running'`).run(new Date().toISOString());
+    for (const { id } of running) recordDeploymentEvent(id, 'Interrupted by application restart. Check the server before retrying.');
 }

src/lib/server/pipeline/worker.ts

@@ -1,3 +1,4 @@
+import { processDestructions } from '../fleet/create.js';
 import { sweep } from './sweep.js';
 import { betaTick } from './beta-runner.js';
 import { emit } from './bus.js';
@@ -27,6 +28,8 @@ function tick(): Promise<void> {
 }
 
 async function runTick(): Promise<void> {
+    try { await processDestructions(); }
+    catch (err) { console.error('[worker] destruction error:', err); }
     try {
         await sweep();
     } catch (err) {

src/lib/ui/components/DestroyServerModal.svelte

@@ -0,0 +1,59 @@
+<script lang="ts">
+    import { goto } from '$app/navigation';
+    import Button from './primitives/Button.svelte';
+
+    let { open = $bindable(false), name, players }: { open?: boolean; name: string; players: number | null } = $props();
+    let dialog = $state<HTMLDialogElement>();
+    let submitting = $state(false);
+    let error = $state('');
+    $effect(() => {
+        if (!dialog) return;
+        if (open && !dialog.open) { error = ''; dialog.showModal(); }
+        else if (!open && dialog.open) dialog.close();
+    });
+
+    async function destroy() {
+        if (submitting) return;
+        submitting = true;
+        error = '';
+        try {
+            const res = await fetch(`/api/servers/${encodeURIComponent(name)}`, { method: 'DELETE' });
+            const body = await res.json();
+            if (!res.ok || !body.ok || !body.deploymentId) throw new Error(body.error ?? 'Could not queue destruction. Please retry.');
+            await goto(`/deployments/${encodeURIComponent(body.deploymentId)}`);
+            open = false;
+        } catch (err) { error = (err as Error).message; }
+        finally { submitting = false; }
+    }
+</script>
+
+<dialog bind:this={dialog} aria-labelledby="destroy-title" onclose={() => open = false}
+    oncancel={(event) => { if (submitting) event.preventDefault(); }}>
+    <form onsubmit={(event) => { event.preventDefault(); void destroy(); }}>
+        <h2 id="destroy-title">Destroy container?</h2>
+        <p class="name">{name}</p>
+        <p>This stops the container, deletes its Komodo deployment, and removes its server files and campaign details. Deployment history and the saved server key are retained.</p>
+        <div class="notice">
+            {#if players !== null && players > 0}<strong>{players} player{players === 1 ? '' : 's'} currently online.</strong>{/if}
+            Destruction waits until a live check confirms the server is empty. You can follow its progress on the deployment page.
+        </div>
+        <p class="irreversible">Once destruction begins, it cannot be undone.</p>
+        {#if error}<p role="alert" class="error">{error}</p>{/if}
+        <footer>
+            <Button variant="ghost" disabled={submitting} onclick={() => open = false}>Cancel</Button>
+            <Button variant="danger" type="submit" loading={submitting}>Destroy container</Button>
+        </footer>
+    </form>
+</dialog>
+
+<style>
+    dialog { width: min(520px, calc(100vw - 32px)); padding: 0; border: 1px solid var(--border-strong); border-radius: 16px; background: var(--surface); color: var(--text-primary); }
+    dialog::backdrop { background: rgb(0 0 0 / .65); backdrop-filter: blur(4px); }
+    form { padding: 28px; } h2 { margin: 0 0 12px; font-size: 22px; }
+    p { line-height: 1.6; color: var(--text-secondary); }
+    .name { font-family: var(--font-mono); color: var(--text-primary); overflow-wrap: anywhere; }
+    .notice { padding: 16px; background: var(--surface-raised); border-radius: 8px; line-height: 1.6; }
+    strong { display: block; color: var(--state-warning); margin-bottom: 4px; }
+    .irreversible { font-size: 13px; } .error { color: var(--state-danger); }
+    footer { display: flex; justify-content: flex-end; flex-wrap: wrap; gap: 12px; margin-top: 24px; }
+</style>