Fix server deletion cleanup and allow failed cleanup to resume

README.md

@@ -78,6 +78,11 @@ are removed, while saved keys and blueprints are retained. The preview must be
 reviewed again if fleet membership changes before confirmation. Servers created
 after confirmation are not included. Progress appears in deployment history.
 
+If deletion fails after Komodo removal, the operation detail page offers
+**Resume cleanup** to retry local file and database removal without RCON or
+repeating Komodo deletion. Requesting destruction again also resumes that cleanup.
+Recovery is blocked if a newer operation exists for the same server name.
+
 Deployments includes a blueprint builder for whole server collections: choose
 multiple T5/T6 maps and set the desired total lobbies for each. **Conform fleet**
 counts existing and queued servers and creates only the shortfall. The preview

src/lib/server/fleet/create.ts

@@ -1,5 +1,5 @@
 import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js';
-import { beginDeployment, deploymentStage, finishDeployment } from './deployments.js';
+import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER, resumeCleanup } from './deployments.js';
 import { randomUUID } from 'node:crypto';
 import type { LaunchPreview } from '$lib/deployment-plan.js';
 import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from './launch-recipes.js';
@@ -423,6 +423,8 @@ function hasActiveCreation(name: string): boolean {
 export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
     const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
     if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
+    const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
+    if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id };
     if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] };
     const operation = db().transaction(() => {
         const id = beginDeployment(name, 'destroy', triggeredBy);
@@ -441,11 +443,22 @@ export function processDestructions(): Promise<void> {
 }
 
 async function runDestructions(): Promise<void> {
-    const queued = db().prepare("SELECT id, server FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string }[];
+    const queued = db().prepare("SELECT id, server, stage FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string; stage: string }[];
     for (const operation of queued) {
         if (creating || destroying.has(operation.server)) continue;
         destroying.add(operation.server);
         try {
+            if (operation.stage === REMOVING_SERVER) {
+                const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create'
+                    AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id);
+                if (newer) {
+                    finishDeployment(operation.id, { ok: false, message: 'A newer creation exists; local cleanup cannot safely resume.' });
+                    continue;
+                }
+                db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(operation.id);
+                finishDeployment(operation.id, await cleanupDestroyedServer(operation.server, operation.id));
+                continue;
+            }
             if (!getServer(operation.server)) {
                 if (hasActiveCreation(operation.server)) {
                     deploymentStage(operation.id, 'Waiting for server creation to finish');
@@ -508,26 +521,35 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
         return { ok: false, kind: 'komodo', message: 'DeleteDeployment failed: ' + deleteRes.message, warnings };
     }
 
-    // 3: host directory. Fall back to derived path if host_dir wasn't set on
+    return cleanupDestroyedServer(name, operation);
+}
+
+async function cleanupDestroyedServer(name: string, operation: string): Promise<DestroyServerResult> {
+    const row = getServer(name);
+    const warnings: string[] = [];
+    // Persist the checkpoint before local work so failed cleanup can resume without RCON.
+    // Legacy operations already recorded this same stage after Komodo deletion.
+    deploymentStage(operation, REMOVING_SERVER);
+    // Host directory. Fall back to derived path if host_dir wasn't set on
     // legacy rows.
-    deploymentStage(operation, 'Removing server files and registry');
-    const hostDir = row.host_dir ?? serverDir(name);
+    const hostDir = row?.host_dir ?? serverDir(name);
     try {
         await rm(hostDir, { recursive: true, force: true });
     } catch (err) {
         warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
     }
 
-    // 4: DB row + cascading rows. ON DELETE CASCADE covers server_scripts,
-    // beta_file_snapshots; the rest (deployment_servers, deployment_files,
-    // beta_servers, beta_server_scripts) reference servers(name) without
-    // cascade — clean them up explicitly.
+    // Remove server-owned beta and deployment rows before the registry row.
+    // Test evidence cascades from runs; inventory and snapshots cascade from servers.
     try {
         const tx = db().transaction((n: string) => {
             db().prepare('DELETE FROM deployment_servers WHERE server = ?').run(n);
             db().prepare('DELETE FROM deployment_files WHERE server = ?').run(n);
             db().prepare('DELETE FROM beta_servers WHERE server = ?').run(n);
             db().prepare('DELETE FROM beta_server_scripts WHERE server = ?').run(n);
+            db().prepare('DELETE FROM beta_test_runs WHERE server = ?').run(n);
+            db().prepare('DELETE FROM beta_test_coverage WHERE server = ?').run(n);
+            db().prepare('DELETE FROM beta_file_changes WHERE server = ?').run(n);
             db().prepare('DELETE FROM servers WHERE name = ?').run(n);
         });
         tx(name);

src/lib/server/fleet/deployments.ts

@@ -30,7 +30,26 @@ export function getDeployment(id: string) {
     const collection = db().prepare(`SELECT c.id, c.name FROM server_collection_members m
         JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ?`).get(id) ?? null;
     return { deployment, events, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
-        canResumeVerification: canResumeVerification(id), collection };
+        canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), collection };
+}
+
+// This stage is written only after Komodo has confirmed deployment deletion.
+export const REMOVING_SERVER = 'Removing server files and registry';
+
+export function canResumeCleanup(id: string): boolean {
+    return !!db().prepare(`SELECT d.id FROM server_deployments d
+        WHERE d.id = ? AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
+        AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
+        .get(id, REMOVING_SERVER);
+}
+
+export function resumeCleanup(id: string): boolean {
+    return db().transaction(() => {
+        if (!canResumeCleanup(id)) return false;
+        db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
+        recordDeploymentEvent(id, 'Resumed local cleanup after confirmed Komodo deletion');
+        return true;
+    })();
 }
 
 export function canResumeVerification(id: string): boolean {

src/routes/api/deployments/[id]/resume/+server.ts

@@ -1,9 +1,9 @@
 import { json } from '@sveltejs/kit';
-import { resumeVerification } from '$lib/server/fleet/deployments.js';
+import { resumeVerification, resumeCleanup } from '$lib/server/fleet/deployments.js';
 import { kick } from '$lib/server/pipeline/worker.js';
 
 export function POST({ params }: { params: { id: string } }) {
-    if (!resumeVerification(params.id)) return json({ error: 'This operation cannot resume verification. Refresh and inspect its server.' }, { status: 409 });
+    if (!resumeVerification(params.id) && !resumeCleanup(params.id)) return json({ error: 'This operation cannot resume. Refresh and inspect its server.' }, { status: 409 });
     void kick().catch(err => console.error('[deployment] verification kick failed:', err));
     return json({ ok: true }, { status: 202 });
 }

src/routes/deployments/[id]/+page.svelte

@@ -14,7 +14,7 @@
         stage: string; started_at: string; finished_at: string | null;
         triggered_by: string; error: string | null; warnings: string; server_exists: number;
     }
-    interface Detail { deployment: Deployment; events: { id: number; created_at: string; message: string }[]; recipe: LaunchRecipeSummary | null; blueprintName: string | null; canResumeVerification: boolean; collection: { id: string; name: string } | null }
+    interface Detail { deployment: Deployment; events: { id: number; created_at: string; message: string }[]; recipe: LaunchRecipeSummary | null; blueprintName: string | null; canResumeVerification: boolean; canResumeCleanup: boolean; collection: { id: string; name: string } | null }
     let detail = $state<Detail | null>(null);
     let error = $state('');
     let refreshing = false;
@@ -34,13 +34,15 @@
         const stages = deployment.action === 'destroy'
             ? ['Starting', 'Waiting', 'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry']
             : ['Starting', 'Preparing server files', 'Creating Komodo deployment', 'Starting container', 'Waiting for Komodo health', 'Waiting for RCON health', 'Waiting for selected map'];
-        if (deployment.state === 'waiting') return 1;
+        if (deployment.state === 'waiting' && deployment.stage !== 'Removing server files and registry') return 1;
         if (deployment.state === 'awaiting-health' && deployment.stage.startsWith('Waiting for health verification')) return 4;
         if (deployment.state === 'done' || deployment.state === 'warning') return stages.length;
         return Math.max(0, stages.findIndex(stage => deployment.stage.startsWith(stage)));
     });
     let explanation = $derived.by(() => {
         if (!deployment) return '';
+        if (detail?.canResumeCleanup) return 'Komodo deletion succeeded. Resume the remaining file and database cleanup; the removed container does not need to answer RCON.';
+        if (deployment.state === 'waiting' && deployment.stage === 'Removing server files and registry') return 'Local cleanup is queued. Komodo deletion has already succeeded.';
         if (deployment.state === 'done') return deployment.action === 'create'
             ? 'The container is healthy, RCON responds, and the reported engine map matches your selection.'
             : 'The container, deployment, and server registration have been removed.';
@@ -68,7 +70,7 @@
         try {
             const res = await fetch(`/api/deployments/${encodeURIComponent(deployment.id)}/resume`, { method: 'POST' });
             const body = await res.json();
-            if (!res.ok) throw new Error(body.error ?? 'Could not resume verification.');
+            if (!res.ok) throw new Error(body.error ?? 'Could not resume the operation.');
             await load();
         } catch (err) { error = (err as Error).message; }
         finally { resuming = false; }
@@ -112,6 +114,7 @@
             <p class="explanation" aria-live="polite">{explanation}</p>
             <div class="timing"><span>Total time <strong>{duration(deployment.started_at, deployment.finished_at ? new Date(deployment.finished_at).getTime() : now)}</strong></span>{#if active && stageStarted}<span>This step <strong>{duration(stageStarted, now)}</strong></span>{/if}</div>
             {#if detail?.canResumeVerification}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume health verification</Button></div>{/if}
+            {#if detail?.canResumeCleanup}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume cleanup</Button></div>{/if}
             {#if deployment.error}<p role="alert" class="error">{deployment.error}</p>{/if}
             {#each warnings as warning}<p class="warning">{warning}</p>{/each}
             {#if deployment.expected_map}<p>Expected map: <strong>{deployment.expected_map}</strong></p>{/if}

tests/unit/create-server.test.ts

@@ -41,6 +41,90 @@ afterEach(async () => {
     vi.unstubAllEnvs();
 });
 
+describe('destruction cleanup recovery', () => {
+    it('removes T6 servers with beta test runs and their dependent evidence', async () => {
+        const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
+        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
+        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
+        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
+        const name = 't6_zombie_town_1';
+        expect(await createServer({ ...input, game: 't6', mapSlug: 'town' })).toMatchObject({ ok: true, name });
+        expect(await createServer(input)).toMatchObject({ ok: true });
+        const { script, version } = createScript({ name: 'test script', kind: 'script', scopeType: 'game', scopeValue: 't6', targetRelPath: 'test.gsc', content: Buffer.from('main(){}') });
+        const beta = db().prepare("INSERT INTO betas (script_id, script_version_id, trigger_kind, state) VALUES (?, ?, 'manual', 'running')").run(script.id, version.id).lastInsertRowid;
+        for (const server of [name, serverName]) {
+            db().prepare("INSERT INTO beta_test_runs (id, beta_id, server, created_at) VALUES (?, ?, ?, '2026-09-18')").run(server, beta, server);
+            db().prepare("INSERT INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, 'event', '2026-09-18', 'health', '{}')").run(server);
+            db().prepare('INSERT INTO beta_test_coverage (beta_id, server) VALUES (?, ?)').run(beta, server);
+            db().prepare("INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, 'test.gsc')").run(beta, server);
+        }
+        const queued = destroyServer(name);
+        await processDestructions();
+        expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
+        for (const table of ['beta_test_runs', 'beta_test_coverage', 'beta_file_changes']) {
+            expect(db().prepare(`SELECT server FROM ${table}`).all()).toEqual([{ server: serverName }]);
+        }
+        expect(db().prepare('SELECT run_id FROM beta_test_evidence').all()).toEqual([{ run_id: serverName }]);
+        expect(db().pragma('foreign_key_check')).toEqual([]);
+    });
+
+    it('resumes failed local cleanup after restart without RCON or another Komodo deletion', async () => {
+        const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
+        const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
+        const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
+        const { POST } = await import('../../src/routes/api/deployments/[id]/resume/+server.js');
+        await createServer(input);
+        db().exec("CREATE TRIGGER fail_cleanup BEFORE DELETE ON servers BEGIN SELECT RAISE(ABORT, 'FOREIGN KEY constraint failed'); END");
+        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
+        const { deploymentId } = destroyServer(serverName);
+        await processDestructions();
+        expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'failed', stage: 'Removing server files and registry', server_exists: 1 });
+        expect(getDeployment(deploymentId!)?.canResumeCleanup).toBe(true);
+        db().exec('DROP TRIGGER fail_cleanup');
+        closeDb();
+        interruptDeployments();
+        vi.mocked(rconGetPlayerCount).mockClear();
+        vi.mocked(komodoDestroyDeployment).mockClear();
+        vi.mocked(komodoDeleteDeployment).mockClear();
+        expect((await POST({ params: { id: deploymentId! } })).status).toBe(202);
+        expect((await POST({ params: { id: deploymentId! } })).status).toBe(409);
+        await processDestructions();
+        expect(rconGetPlayerCount).not.toHaveBeenCalled();
+        expect(komodoDestroyDeployment).not.toHaveBeenCalled();
+        expect(komodoDeleteDeployment).not.toHaveBeenCalled();
+        expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
+    });
+
+    it('reuses an interrupted cleanup when destruction is requested again', async () => {
+        const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
+        const { deploymentStage, interruptDeployments, getDeployment, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        await createServer(input);
+        const first = destroyServer(serverName);
+        deploymentStage(first.deploymentId!, REMOVING_SERVER);
+        db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(first.deploymentId);
+        interruptDeployments();
+        expect(destroyServer(serverName).deploymentId).toBe(first.deploymentId);
+        expect(getDeployment(first.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: REMOVING_SERVER });
+    });
+
+    it('refuses cleanup before confirmed deletion or after a newer creation', async () => {
+        const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
+        const { beginDeployment, deploymentStage, finishDeployment, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
+        await createServer(input);
+        const { deploymentId } = destroyServer(serverName);
+        finishDeployment(deploymentId!, { ok: false, message: 'Komodo unavailable' });
+        expect(resumeCleanup(deploymentId!)).toBe(false);
+        deploymentStage(deploymentId!, REMOVING_SERVER);
+        beginDeployment(serverName, 'create');
+        expect(resumeCleanup(deploymentId!)).toBe(false);
+    });
+});
+
 describe('House Party Protocol', () => {
     it('includes registered and queued servers once and reuses destruction requests', async () => {
         const { createServer, queueCreateServer } = await import('../../src/lib/server/fleet/create.js');