Add configurable server destruction concurrency

README.md

@@ -118,7 +118,8 @@ for verification, retry, scheduling, and compatibility details.
 ## Fleet settings
 
 **Settings → Fleet settings** controls campaign history retention (default 7 days),
-concurrent restarts (default 1), and concurrent server startups (default 1).
+concurrent restarts (default 1), concurrent server startups (default 1), and
+concurrent server destructions (default 1).
 Values are positive whole numbers; retention supports up to 36500 days. Settings
 are saved in SQLite for all users and apply at the next queue or cleanup check
 without restarting the app. Lowering a limit leaves active operations running
@@ -134,9 +135,11 @@ On the first startup after upgrading, existing `DEPLOY_KEEP_DAYS`,
 values initialize the saved settings. Keep custom values for that first startup;
 afterward, the database values take precedence and those variables can be removed.
 The supplied Compose and Komodo examples use the built-in defaults.
+Existing saved settings retain their values when upgraded; the destruction
+limit starts at 1 and can be changed in Settings.
 
-`GET /api/settings` reads the three values (`deployKeepDays`, `restartConcurrency`,
-`createConcurrency`); `PUT /api/settings` saves all three together as JSON.
+`GET /api/settings` reads the four values (`deployKeepDays`, `restartConcurrency`,
+`createConcurrency`, `destroyConcurrency`); `PUT /api/settings` saves all four together as JSON.
 Invalid or incomplete requests return HTTP 400 without changing saved settings.
 
 ## How restarts stay safe
@@ -447,6 +450,10 @@ when creation began. The page retains its last snapshot if a refresh fails.
 
 Destroy uses an in-app confirmation modal and immediately returns a queued deployment (`DELETE /api/servers/:name` returns HTTP 202 with `deploymentId`). The worker waits for a fresh zero-player RCON or telemetry report, or a verified stopped container, before teardown. Occupied reports from either source block teardown; servers without current verification remain queued and are checked each worker tick. Waiting requests survive application restarts, while interrupted teardown is marked for inspection. Completed destruction stays on the provisioning detail page.
 
+**Settings → Concurrent server destructions** limits simultaneous individual and
+fleet removals (default 1). Each destruction holds its slot through teardown and
+local cleanup. Lowering the limit lets active removals finish before more begin.
+
 
 Use **Provisioning → Manage server**, or a server’s **Server actions**, to start,
 restart, stop, or rebuild an existing instance. **Rebuild destroys and creates in

src/lib/fleet-settings.ts

@@ -2,18 +2,21 @@ export interface FleetSettings {
     deployKeepDays: number;
     restartConcurrency: number;
     createConcurrency: number;
+    destroyConcurrency: number;
 }
 
 export const defaultFleetSettings: FleetSettings = {
     deployKeepDays: 7,
     restartConcurrency: 1,
-    createConcurrency: 1
+    createConcurrency: 1,
+    destroyConcurrency: 1
 };
 
 export const fleetSettingLabels: Record<keyof FleetSettings, string> = {
     deployKeepDays: 'Campaign history retention',
     restartConcurrency: 'Concurrent restarts',
-    createConcurrency: 'Concurrent server startups'
+    createConcurrency: 'Concurrent server startups',
+    destroyConcurrency: 'Concurrent server destructions'
 };
 
 export function validateFleetSettings(input: unknown): FleetSettings {
@@ -30,5 +33,5 @@ export function validateFleetSettings(input: unknown): FleetSettings {
         if (key === 'deployKeepDays' && value > 36500) throw new Error('Campaign history retention must be at most 36500 days.');
     }
     return { deployKeepDays: values.deployKeepDays as number, restartConcurrency: values.restartConcurrency as number,
-        createConcurrency: values.createConcurrency as number };
+        createConcurrency: values.createConcurrency as number, destroyConcurrency: values.destroyConcurrency as number };
 }

src/lib/server/db/fleet-settings.ts

@@ -5,14 +5,22 @@ export const FLEET_SETTINGS_KEY = 'fleet_settings';
 
 /** Import legacy environment values once; persisted settings win on every later boot. */
 export function initializeFleetSettings(handle: Database.Database): void {
-    if (handle.prepare('SELECT 1 FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY)) return;
+    const saved = handle.prepare('SELECT value FROM settings WHERE key = ?').get(FLEET_SETTINGS_KEY) as { value: string } | undefined;
+    if (saved) {
+        const settings = JSON.parse(saved.value);
+        if (!Object.hasOwn(settings, 'destroyConcurrency')) {
+            const upgraded = validateFleetSettings({ ...settings, destroyConcurrency: defaultFleetSettings.destroyConcurrency });
+            handle.prepare('UPDATE settings SET value = ? WHERE key = ?').run(JSON.stringify(upgraded), FLEET_SETTINGS_KEY);
+        }
+        return;
+    }
     const values = { ...defaultFleetSettings };
-    const legacy: Record<keyof FleetSettings, string> = {
+    const legacy: Partial<Record<keyof FleetSettings, string>> = {
         deployKeepDays: 'DEPLOY_KEEP_DAYS', restartConcurrency: 'DEPLOYMENT_RESTART_CONCURRENCY',
         createConcurrency: 'DEPLOYMENT_CREATE_CONCURRENCY'
     };
     for (const key of Object.keys(legacy) as (keyof FleetSettings)[]) {
-        const raw = process.env[legacy[key]];
+        const raw = process.env[legacy[key]!];
         if (raw) values[key] = Number(raw);
     }
     handle.prepare('INSERT OR IGNORE INTO settings (key, value) VALUES (?, ?)')

src/lib/server/provisioning/instances.ts

@@ -570,83 +570,105 @@ function betaRemovalEvidence(server: string, message: string): ProvisioningBlock
     return { code: 'active-beta', message, automatic: true, href: beta ? `/betas/${beta.id}` : null, label: 'Blocking beta' };
 }
 
+interface DestructionJob { id: string; server: string; stage: string; server_deleted_at: string | null }
+
 async function runDestructions(): Promise<void> {
     const queued = db().prepare(`SELECT id, server, stage, server_deleted_at FROM server_deployments WHERE action = 'destroy' AND state = 'waiting'
-        AND (server_deleted_at IS NULL OR stage = ?) ORDER BY started_at`).all(REMOVING_SERVER) as { id: string; server: string; stage: string; server_deleted_at: string | null }[];
-    for (const operation of queued) {
-        if (creating || destroying.has(operation.server)) continue;
-        if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
-        destroying.add(operation.server);
-        try {
-            if (operation.stage === REMOVING_SERVER) {
-                if (operation.server_deleted_at && getServer(operation.server)) {
-                    finishDeployment(operation.id, { ok: false, message: 'A replacement server is registered; local cleanup cannot safely resume.' });
-                    continue;
-                }
-                const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create'
-                    AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id);
-                if (newer) {
-                    finishDeployment(operation.id, { ok: false, message: 'A newer creation exists; local cleanup cannot safely resume.' });
-                    continue;
-                }
-                if (!claimRemoval(operation.id)) continue;
-                finishDeployment(operation.id, await cleanupDestroyedServer(operation.server, operation.id));
-                continue;
+        AND (server_deleted_at IS NULL OR stage = ?) ORDER BY started_at, rowid`).all(REMOVING_SERVER) as DestructionJob[];
+    const active = new Set<Promise<void>>();
+    let next = 0;
+    try {
+        while (next < queued.length || active.size) {
+            // Recheck the saved limit before each admission. Lowering it never
+            // interrupts an active removal; its slot lasts through local cleanup.
+            while (next < queued.length && !creating && destroying.size < fleetSettings().destroyConcurrency
+                && active.size < fleetSettings().destroyConcurrency) {
+                const task = processDestruction(queued[next++]).finally(() => { active.delete(task); });
+                active.add(task);
             }
-            if (!getServer(operation.server)) {
-                if (hasActiveCreation(operation.server)) {
-                    const creatingOperation = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL
-                        AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health') LIMIT 1`).get(operation.server) as { id: string } | undefined;
-                    deploymentStage(operation.id, 'Waiting for server creation to finish', { phase: 'eligibility', checks: [],
-                        blocker: { code: 'creation-pending', message: 'Waiting for server creation to finish', automatic: true,
-                            href: creatingOperation ? `/provisioning/${encodeURIComponent(creatingOperation.id)}` : null, label: 'Creation operation' } });
-                    continue;
-                }
-                finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' });
-                continue;
-            }
-            const blocker = activeBetaRemovalBlocker(operation.server);
-            if (blocker) { deploymentStage(operation.id, blocker, { phase: 'eligibility', checks: [], blocker: betaRemovalEvidence(operation.server, blocker) }); continue; }
-            // A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
-            const evidence = await probePlayerEvidence(operation.server);
-            let players = evidence.players;
-            let occupancy: NonNullable<ProgressUpdate['checks']>[number] = { key: 'occupancy', label: 'Player occupancy',
-                result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting', source: evidence.source,
-                checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players };
-            if (players === null) {
-                const health = await komodoInspectHealthy(operation.server);
-                if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) {
-                    players = 0;
-                    occupancy = { key: 'stopped', label: 'Stopped container permits removal', result: 'pass', source: 'komodo', actual: health.state ?? null };
-                }
-            }
-            if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
-            // Beta ownership may have changed while the network probes were in flight.
-            const latestBlocker = activeBetaRemovalBlocker(operation.server);
-            if (latestBlocker) { deploymentStage(operation.id, latestBlocker, { phase: 'eligibility', checks: [occupancy], blocker: betaRemovalEvidence(operation.server, latestBlocker) }); continue; }
-            if (players !== 0) {
-                const message = players === null
-                    ? 'Waiting for player count: server could not be reached'
-                    : 'Waiting for server to empty: ' + players + ' player(s) online';
-                deploymentStage(operation.id, message, { phase: 'eligibility', checks: [occupancy],
-                    blocker: { code: players === null ? 'players-unknown' : 'players-present', message, automatic: true } });
-                continue;
-            }
-            if (!claimRemoval(operation.id)) continue;
-            deploymentStage(operation.id, 'Server is empty (0 players verified)', { phase: 'eligibility', checks: [occupancy], blocker: null,
-                checkpoint: { key: 'eligibility', label: 'Removal eligibility verified' } });
-            const result = await destroyServerUnlocked(operation.server, operation.id);
-            finishDeployment(operation.id, result);
-        } catch (err) {
-            const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string };
-            if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message, {
-                phase: 'eligibility', checks: [], blocker: { code: 'players-unknown', message: (err as Error).message, automatic: true }
-            });
-            else if (row.state !== 'cancelled') finishDeployment(operation.id, { ok: false, message: (err as Error).message });
-        } finally { destroying.delete(operation.server); }
+            if (!active.size) break; // Creation or rebuilding holds the remaining capacity.
+            await Promise.race(active);
+        }
+    } finally {
+        // Keep the shared pass locked until every admitted removal has settled,
+        // including when an unexpected persistence error escapes a task.
+        await Promise.allSettled(active);
     }
 }
 
+async function processDestruction(operation: DestructionJob): Promise<void> {
+    if (creating || destroying.has(operation.server)) return;
+    if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) return;
+    destroying.add(operation.server);
+    try {
+        if (operation.stage === REMOVING_SERVER) {
+            if (operation.server_deleted_at && getServer(operation.server)) {
+                finishDeployment(operation.id, { ok: false, message: 'A replacement server is registered; local cleanup cannot safely resume.' });
+                return;
+            }
+            const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create'
+                AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id);
+            if (newer) {
+                finishDeployment(operation.id, { ok: false, message: 'A newer creation exists; local cleanup cannot safely resume.' });
+                return;
+            }
+            if (!claimRemoval(operation.id)) return;
+            finishDeployment(operation.id, await cleanupDestroyedServer(operation.server, operation.id));
+            return;
+        }
+        if (!getServer(operation.server)) {
+            if (hasActiveCreation(operation.server)) {
+                const creatingOperation = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL
+                    AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health') LIMIT 1`).get(operation.server) as { id: string } | undefined;
+                deploymentStage(operation.id, 'Waiting for server creation to finish', { phase: 'eligibility', checks: [],
+                    blocker: { code: 'creation-pending', message: 'Waiting for server creation to finish', automatic: true,
+                        href: creatingOperation ? `/provisioning/${encodeURIComponent(creatingOperation.id)}` : null, label: 'Creation operation' } });
+                return;
+            }
+            finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' });
+            return;
+        }
+        const blocker = activeBetaRemovalBlocker(operation.server);
+        if (blocker) { deploymentStage(operation.id, blocker, { phase: 'eligibility', checks: [], blocker: betaRemovalEvidence(operation.server, blocker) }); return; }
+        // A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
+        const evidence = await probePlayerEvidence(operation.server);
+        let players = evidence.players;
+        let occupancy: NonNullable<ProgressUpdate['checks']>[number] = { key: 'occupancy', label: 'Player occupancy',
+            result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting', source: evidence.source,
+            checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players };
+        if (players === null) {
+            const health = await komodoInspectHealthy(operation.server);
+            if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) {
+                players = 0;
+                occupancy = { key: 'stopped', label: 'Stopped container permits removal', result: 'pass', source: 'komodo', actual: health.state ?? null };
+            }
+        }
+        if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) return;
+        // Beta ownership may have changed while the network probes were in flight.
+        const latestBlocker = activeBetaRemovalBlocker(operation.server);
+        if (latestBlocker) { deploymentStage(operation.id, latestBlocker, { phase: 'eligibility', checks: [occupancy], blocker: betaRemovalEvidence(operation.server, latestBlocker) }); return; }
+        if (players !== 0) {
+            const message = players === null
+                ? 'Waiting for player count: server could not be reached'
+                : 'Waiting for server to empty: ' + players + ' player(s) online';
+            deploymentStage(operation.id, message, { phase: 'eligibility', checks: [occupancy],
+                blocker: { code: players === null ? 'players-unknown' : 'players-present', message, automatic: true } });
+            return;
+        }
+        if (!claimRemoval(operation.id)) return;
+        deploymentStage(operation.id, 'Server is empty (0 players verified)', { phase: 'eligibility', checks: [occupancy], blocker: null,
+            checkpoint: { key: 'eligibility', label: 'Removal eligibility verified' } });
+        const result = await destroyServerUnlocked(operation.server, operation.id);
+        finishDeployment(operation.id, result);
+    } catch (err) {
+        const row = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(operation.id) as { state: string };
+        if (row.state === 'waiting') deploymentStage(operation.id, 'Waiting for player count: ' + (err as Error).message, {
+            phase: 'eligibility', checks: [], blocker: { code: 'players-unknown', message: (err as Error).message, automatic: true }
+        });
+        else if (row.state !== 'cancelled') finishDeployment(operation.id, { ok: false, message: (err as Error).message });
+    } finally { destroying.delete(operation.server); }
+}
+
 async function destroyServerUnlocked(name: string, operation: string): Promise<DestroyServerResult> {
     const blocker = activeBetaRemovalBlocker(name);
     if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] };

src/routes/settings/+page.svelte

@@ -60,6 +60,9 @@
                     <label for="startups">Concurrent server startups</label>
                     <input id="startups" type="number" min="1" step="1" required bind:value={settings.createConcurrency} aria-describedby="startups-help" />
                     <p id="startups-help">Default: 1 across collections and individual launches. Each startup holds its slot through container, game health, and selected-map verification.</p>
+                    <label for="destructions">Concurrent server destructions</label>
+                    <input id="destructions" type="number" min="1" step="1" required bind:value={settings.destroyConcurrency} aria-describedby="destructions-help" />
+                    <p id="destructions-help">Default: 1 across individual and fleet removals. Each destruction holds its slot until teardown and local cleanup finish.</p>
                     <p>Lowering a limit lets active operations finish; queued work waits for capacity. Player safety checks still apply.</p>
                     <Button type="submit" disabled={saving}>{saving ? 'Saving…' : 'Save fleet settings'}</Button>
                 </fieldset>

tests/unit/destruction-concurrency.test.ts

@@ -0,0 +1,215 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { join } from 'node:path';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
+    komodoCreateDeployment: vi.fn(),
+    komodoDeploy: vi.fn(),
+    komodoDestroyDeployment: vi.fn(),
+    komodoDeleteDeployment: vi.fn(),
+    komodoInspectHealthy: vi.fn(),
+    komodoEnsureTag: vi.fn(),
+    komodoUpdateTagsOnResource: vi.fn()
+}));
+vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
+    rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
+}));
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
+
+function deferred() {
+    let resolve!: () => void;
+    let reject!: (error: Error) => void;
+    const promise = new Promise<void>((done, fail) => { resolve = done; reject = fail; });
+    return { promise, resolve, reject };
+}
+
+function command() {
+    return { started: deferred(), response: deferred() };
+}
+
+const servers = Array.from({ length: 4 }, (_, index) => `t5_zombie_theater_${index + 1}`);
+let fixture: Fixture;
+let commands: Map<string, { destroy: ReturnType<typeof command>; delete: ReturnType<typeof command> }>;
+let passes: Promise<void>[];
+let runningAtSubmission: number[];
+
+beforeEach(async () => {
+    vi.resetModules(); vi.resetAllMocks();
+    fixture = await newFixture(servers);
+    for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test',
+        KOMODO_API_SECRET: 'test', KOMODO_TARGET_SERVER: 'test-host', RCON_PASSWORD: 'test' })) vi.stubEnv(key, value);
+    commands = new Map(servers.map(server => [server, { destroy: command(), delete: command() }]));
+    passes = [];
+    runningAtSubmission = [];
+    const { db } = await import('../../src/lib/server/db/index.js');
+    for (const [index, server] of servers.entries()) {
+        db().prepare(`INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, game_port, host_dir)
+            VALUES (?, 'deployment', ?, 't5', 'zombie_theater', 'zombie', 'theater', ?, ?)`)
+            .run(server, server, 28960 + index, join(fixture.t5, server));
+    }
+    const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
+    const { komodoDestroyDeployment, komodoDeleteDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
+    vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
+    vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
+    vi.mocked(komodoDestroyDeployment).mockImplementation(async server => {
+        runningAtSubmission.push((db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action = 'destroy' AND state = 'running'")
+            .get() as { n: number }).n);
+        const gate = commands.get(server)!.destroy;
+        gate.started.resolve();
+        await gate.response.promise;
+        return { ok: true };
+    });
+    vi.mocked(komodoDeleteDeployment).mockImplementation(async server => {
+        const gate = commands.get(server)!.delete;
+        gate.started.resolve();
+        await gate.response.promise;
+        return { ok: true };
+    });
+});
+
+afterEach(async () => {
+    for (const server of servers) complete(server);
+    await Promise.allSettled(passes);
+    await fixture.cleanup();
+    vi.unstubAllEnvs();
+    vi.restoreAllMocks();
+});
+
+function complete(server: string) {
+    commands.get(server)!.destroy.response.resolve();
+    commands.get(server)!.delete.response.resolve();
+}
+
+function track(pass: Promise<void>) {
+    passes.push(pass);
+    return pass;
+}
+
+async function queue(count = servers.length) {
+    const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
+    return servers.slice(0, count).map(server => {
+        const result = destroyServer(server);
+        if (!result.ok) throw new Error(result.message);
+        return result.deploymentId!;
+    });
+}
+
+async function states() {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    return (db().prepare("SELECT state FROM server_deployments WHERE action = 'destroy' ORDER BY rowid")
+        .all() as { state: string }[]).map(row => row.state);
+}
+
+describe('destruction concurrency', () => {
+    it('defaults to one full removal at a time and refills the slot in the same pass', async () => {
+        await queue(2);
+        const { fleetSettings } = await import('../../src/lib/server/settings.js');
+        const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
+        const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
+        expect(fleetSettings().destroyConcurrency).toBe(1);
+        const pass = track(processDestructions());
+        await commands.get(servers[0])!.destroy.started.promise;
+        expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
+        expect(await states()).toEqual(['running', 'waiting']);
+        complete(servers[0]);
+        await commands.get(servers[1])!.destroy.started.promise;
+        expect(await states()).toEqual(['done', 'running']);
+        complete(servers[1]);
+        await pass;
+        expect(await states()).toEqual(['done', 'done']);
+        expect(runningAtSubmission).toEqual([1, 1]);
+    });
+
+    it('runs two removals concurrently, holds slots through deletion, and coalesces overlapping passes', async () => {
+        await queue();
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
+        const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
+        const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
+        const pass = track(processDestructions());
+        await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise));
+        const overlapping = track(processDestructions());
+        expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2);
+        expect(await states()).toEqual(['running', 'running', 'waiting', 'waiting']);
+        commands.get(servers[0])!.destroy.response.resolve();
+        await commands.get(servers[0])!.delete.started.promise;
+        expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2);
+        commands.get(servers[0])!.delete.response.resolve();
+        await commands.get(servers[2])!.destroy.started.promise;
+        expect(await states()).toEqual(['done', 'running', 'running', 'waiting']);
+        complete(servers[1]);
+        await commands.get(servers[3])!.destroy.started.promise;
+        complete(servers[2]); complete(servers[3]);
+        await Promise.all([pass, overlapping]);
+        expect(await states()).toEqual(['done', 'done', 'done', 'done']);
+        expect(vi.mocked(komodoDestroyDeployment).mock.calls.map(([server]) => server)).toEqual(servers);
+        expect(komodoDeleteDeployment).toHaveBeenCalledTimes(4);
+        expect(Math.max(...runningAtSubmission)).toBe(2);
+    });
+
+    it('lets active removals finish after lowering the limit and applies it to new admissions', async () => {
+        await queue();
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
+        const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
+        const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
+        const pass = track(processDestructions());
+        await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise));
+        saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 1 });
+        complete(servers[0]);
+        await vi.waitFor(async () => expect(await states()).toEqual(['done', 'running', 'waiting', 'waiting']));
+        expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2);
+        complete(servers[1]);
+        await commands.get(servers[2])!.destroy.started.promise;
+        expect(komodoDestroyDeployment).toHaveBeenCalledTimes(3);
+        expect(await states()).toEqual(['done', 'done', 'running', 'waiting']);
+        complete(servers[2]);
+        await commands.get(servers[3])!.destroy.started.promise;
+        complete(servers[3]);
+        await pass;
+        expect(runningAtSubmission.slice(2)).toEqual([1, 1]);
+    });
+
+    it.each(['rejected', 'thrown'] as const)('releases capacity after a %s teardown failure', async failure => {
+        await queue(3);
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
+        const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
+        const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
+        if (failure === 'rejected') {
+            vi.mocked(komodoDestroyDeployment).mockImplementationOnce(async () => {
+                commands.get(servers[0])!.destroy.started.resolve();
+                await commands.get(servers[0])!.destroy.response.promise;
+                return { ok: false, kind: 'http-error', message: 'Teardown rejected' };
+            });
+        }
+        const pass = track(processDestructions());
+        await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise));
+        if (failure === 'thrown') commands.get(servers[0])!.destroy.response.reject(new Error('Connection lost'));
+        else commands.get(servers[0])!.destroy.response.resolve();
+        await commands.get(servers[2])!.destroy.started.promise;
+        expect(await states()).toEqual(['failed', 'running', 'running']);
+        complete(servers[1]); complete(servers[2]);
+        await pass;
+        expect(await states()).toEqual(['failed', 'done', 'done']);
+        expect(komodoDestroyDeployment).toHaveBeenCalledTimes(3);
+        expect(komodoDeleteDeployment).not.toHaveBeenCalledWith(servers[0]);
+    });
+
+    it('skips occupied and unknown servers while allowing later empty and stopped servers', async () => {
+        await queue();
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
+        const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
+        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
+        const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
+        vi.mocked(rconGetPlayerCount).mockImplementation(async server => server === servers[0] ? 2 : server === servers[2] ? 0 : null);
+        vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === servers[1]
+            ? { ok: false, reason: 'Host unavailable' } : { ok: false, state: 'exited', reason: 'stopped' });
+        for (const server of servers) complete(server);
+        await track(processDestructions());
+        expect(await states()).toEqual(['waiting', 'waiting', 'done', 'done']);
+        expect(vi.mocked(komodoDestroyDeployment).mock.calls.map(([server]) => server)).toEqual(servers.slice(2));
+        expect(rconGetPlayerCount).toHaveBeenCalledTimes(4);
+    });
+});

tests/unit/fleet-settings.test.ts

@@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { newFixture, type Fixture } from '../helpers/fixture.js';
 
 let fixture: Fixture;
-const defaults = { deployKeepDays: 7, restartConcurrency: 1, createConcurrency: 1 };
+const defaults = { deployKeepDays: 7, restartConcurrency: 1, createConcurrency: 1, destroyConcurrency: 1 };
 beforeEach(async () => {
     vi.resetModules();
     fixture = await newFixture([]);
@@ -14,7 +14,7 @@ describe('fleet settings API and persistence', () => {
     it('returns defaults and saves all settings across a module reload and database reopen', async () => {
         const { GET, PUT } = await import('../../src/routes/api/settings/+server.js');
         expect(await GET().json()).toEqual(defaults);
-        const saved = { deployKeepDays: 30, restartConcurrency: 3, createConcurrency: 2 };
+        const saved = { deployKeepDays: 30, restartConcurrency: 3, createConcurrency: 2, destroyConcurrency: 4 };
         const response = await PUT({ request: new Request('http://localhost/api/settings', { method: 'PUT', body: JSON.stringify(saved) }) });
         expect(response.status).toBe(200);
         expect(await response.json()).toEqual(saved);
@@ -37,17 +37,34 @@ describe('fleet settings API and persistence', () => {
         vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '4');
         vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
         const { fleetSettings } = await import('../../src/lib/server/settings.js');
-        expect(fleetSettings()).toEqual({ deployKeepDays: 14, restartConcurrency: 4, createConcurrency: 2 });
+        expect(fleetSettings()).toEqual({ deployKeepDays: 14, restartConcurrency: 4, createConcurrency: 2, destroyConcurrency: 1 });
         closeDb();
         vi.stubEnv('DEPLOY_KEEP_DAYS', '7');
         expect(fleetSettings().deployKeepDays).toBe(14);
         expect(db().prepare("SELECT value FROM settings WHERE key = 'unrelated'").get()).toEqual({ value: 'true' });
     });
 
+    it('upgrades saved settings with a default destruction limit while retaining existing values', async () => {
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        const previous = { deployKeepDays: 21, restartConcurrency: 3, createConcurrency: 2 };
+        db().prepare("UPDATE settings SET value = ? WHERE key = 'fleet_settings'").run(JSON.stringify(previous));
+        closeDb();
+        vi.stubEnv('DEPLOY_KEEP_DAYS', 'bad');
+        vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '9');
+        vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '8');
+        const { GET } = await import('../../src/routes/api/settings/+server.js');
+        const upgraded = { ...previous, destroyConcurrency: 1 };
+        expect(await GET().json()).toEqual(upgraded);
+        expect(JSON.parse((db().prepare("SELECT value FROM settings WHERE key = 'fleet_settings'").get() as { value: string }).value)).toEqual(upgraded);
+        closeDb();
+        expect(await GET().json()).toEqual(upgraded);
+    });
+
     it.each([
         'null', '[]', '{}', '{',
-        ...['deployKeepDays', 'restartConcurrency', 'createConcurrency'].flatMap(key =>
+        ...['deployKeepDays', 'restartConcurrency', 'createConcurrency', 'destroyConcurrency'].flatMap(key =>
             [0, -1, 1.5, '2', true, null, 9007199254740992].map(value => JSON.stringify({ ...defaults, [key]: value }))),
+        JSON.stringify({ deployKeepDays: 7, restartConcurrency: 1, createConcurrency: 1 }),
         JSON.stringify({ ...defaults, deployKeepDays: 36501 }),
         JSON.stringify({ ...defaults, unknown: 1 })
     ])('rejects invalid input without partially saving: %s', async body => {