Fix server deletion cleanup and allow failed cleanup to resume

tests/unit/create-server.test.ts

@@ -41,6 +41,90 @@ afterEach(async () => {
     vi.unstubAllEnvs();
 });
 
+describe('destruction cleanup recovery', () => {
+    it('removes T6 servers with beta test runs and their dependent evidence', async () => {
+        const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
+        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
+        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
+        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
+        const name = 't6_zombie_town_1';
+        expect(await createServer({ ...input, game: 't6', mapSlug: 'town' })).toMatchObject({ ok: true, name });
+        expect(await createServer(input)).toMatchObject({ ok: true });
+        const { script, version } = createScript({ name: 'test script', kind: 'script', scopeType: 'game', scopeValue: 't6', targetRelPath: 'test.gsc', content: Buffer.from('main(){}') });
+        const beta = db().prepare("INSERT INTO betas (script_id, script_version_id, trigger_kind, state) VALUES (?, ?, 'manual', 'running')").run(script.id, version.id).lastInsertRowid;
+        for (const server of [name, serverName]) {
+            db().prepare("INSERT INTO beta_test_runs (id, beta_id, server, created_at) VALUES (?, ?, ?, '2026-09-18')").run(server, beta, server);
+            db().prepare("INSERT INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, 'event', '2026-09-18', 'health', '{}')").run(server);
+            db().prepare('INSERT INTO beta_test_coverage (beta_id, server) VALUES (?, ?)').run(beta, server);
+            db().prepare("INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, 'test.gsc')").run(beta, server);
+        }
+        const queued = destroyServer(name);
+        await processDestructions();
+        expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
+        for (const table of ['beta_test_runs', 'beta_test_coverage', 'beta_file_changes']) {
+            expect(db().prepare(`SELECT server FROM ${table}`).all()).toEqual([{ server: serverName }]);
+        }
+        expect(db().prepare('SELECT run_id FROM beta_test_evidence').all()).toEqual([{ run_id: serverName }]);
+        expect(db().pragma('foreign_key_check')).toEqual([]);
+    });
+
+    it('resumes failed local cleanup after restart without RCON or another Komodo deletion', async () => {
+        const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
+        const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
+        const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
+        const { POST } = await import('../../src/routes/api/deployments/[id]/resume/+server.js');
+        await createServer(input);
+        db().exec("CREATE TRIGGER fail_cleanup BEFORE DELETE ON servers BEGIN SELECT RAISE(ABORT, 'FOREIGN KEY constraint failed'); END");
+        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
+        const { deploymentId } = destroyServer(serverName);
+        await processDestructions();
+        expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'failed', stage: 'Removing server files and registry', server_exists: 1 });
+        expect(getDeployment(deploymentId!)?.canResumeCleanup).toBe(true);
+        db().exec('DROP TRIGGER fail_cleanup');
+        closeDb();
+        interruptDeployments();
+        vi.mocked(rconGetPlayerCount).mockClear();
+        vi.mocked(komodoDestroyDeployment).mockClear();
+        vi.mocked(komodoDeleteDeployment).mockClear();
+        expect((await POST({ params: { id: deploymentId! } })).status).toBe(202);
+        expect((await POST({ params: { id: deploymentId! } })).status).toBe(409);
+        await processDestructions();
+        expect(rconGetPlayerCount).not.toHaveBeenCalled();
+        expect(komodoDestroyDeployment).not.toHaveBeenCalled();
+        expect(komodoDeleteDeployment).not.toHaveBeenCalled();
+        expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
+    });
+
+    it('reuses an interrupted cleanup when destruction is requested again', async () => {
+        const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
+        const { deploymentStage, interruptDeployments, getDeployment, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        await createServer(input);
+        const first = destroyServer(serverName);
+        deploymentStage(first.deploymentId!, REMOVING_SERVER);
+        db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(first.deploymentId);
+        interruptDeployments();
+        expect(destroyServer(serverName).deploymentId).toBe(first.deploymentId);
+        expect(getDeployment(first.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: REMOVING_SERVER });
+    });
+
+    it('refuses cleanup before confirmed deletion or after a newer creation', async () => {
+        const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
+        const { beginDeployment, deploymentStage, finishDeployment, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
+        await createServer(input);
+        const { deploymentId } = destroyServer(serverName);
+        finishDeployment(deploymentId!, { ok: false, message: 'Komodo unavailable' });
+        expect(resumeCleanup(deploymentId!)).toBe(false);
+        deploymentStage(deploymentId!, REMOVING_SERVER);
+        beginDeployment(serverName, 'create');
+        expect(resumeCleanup(deploymentId!)).toBe(false);
+    });
+});
+
 describe('House Party Protocol', () => {
     it('includes registered and queued servers once and reuses destruction requests', async () => {
         const { createServer, queueCreateServer } = await import('../../src/lib/server/fleet/create.js');