import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(), komodoInspectHealthy: vi.fn(), komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn() })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); function deferred() { let resolve!: () => void; let reject!: (error: Error) => void; const promise = new Promise((done, fail) => { resolve = done; reject = fail; }); return { promise, resolve, reject }; } function command() { return { started: deferred(), response: deferred() }; } const servers = Array.from({ length: 4 }, (_, index) => `t5_zombie_theater_${index + 1}`); let fixture: Fixture; let commands: Map; delete: ReturnType }>; let passes: Promise[]; let runningAtSubmission: number[]; beforeEach(async () => { vi.resetModules(); vi.resetAllMocks(); fixture = await newFixture(servers); for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test', KOMODO_TARGET_SERVER: 'test-host', RCON_PASSWORD: 'test' })) vi.stubEnv(key, value); commands = new Map(servers.map(server => [server, { destroy: command(), delete: command() }])); passes = []; runningAtSubmission = []; const { db } = await import('../../src/lib/server/db/index.js'); for (const [index, server] of servers.entries()) { db().prepare(`INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, game_port, host_dir) VALUES (?, 'deployment', ?, 't5', 'zombie_theater', 'zombie', 'theater', ?, ?)`) .run(server, server, 28960 + index, join(fixture.t5, server)); } const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); const { komodoDestroyDeployment, komodoDeleteDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(rconGetPlayerCount).mockResolvedValue(0); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' }); vi.mocked(komodoDestroyDeployment).mockImplementation(async server => { runningAtSubmission.push((db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action = 'destroy' AND state = 'running'") .get() as { n: number }).n); const gate = commands.get(server)!.destroy; gate.started.resolve(); await gate.response.promise; return { ok: true }; }); vi.mocked(komodoDeleteDeployment).mockImplementation(async server => { const gate = commands.get(server)!.delete; gate.started.resolve(); await gate.response.promise; return { ok: true }; }); }); afterEach(async () => { for (const server of servers) complete(server); await Promise.allSettled(passes); await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); }); function complete(server: string) { commands.get(server)!.destroy.response.resolve(); commands.get(server)!.delete.response.resolve(); } function track(pass: Promise) { passes.push(pass); return pass; } async function queue(count = servers.length) { const { destroyServer } = await import('../../src/lib/server/fleet/create.js'); return servers.slice(0, count).map(server => { const result = destroyServer(server); if (!result.ok) throw new Error(result.message); return result.deploymentId!; }); } async function states() { const { db } = await import('../../src/lib/server/db/index.js'); return (db().prepare("SELECT state FROM server_deployments WHERE action = 'destroy' ORDER BY rowid") .all() as { state: string }[]).map(row => row.state); } describe('destruction concurrency', () => { it('defaults to one full removal at a time and refills the slot in the same pass', async () => { await queue(2); const { fleetSettings } = await import('../../src/lib/server/settings.js'); const { processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); expect(fleetSettings().destroyConcurrency).toBe(1); const pass = track(processDestructions()); await commands.get(servers[0])!.destroy.started.promise; expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1); expect(await states()).toEqual(['running', 'waiting']); complete(servers[0]); await commands.get(servers[1])!.destroy.started.promise; expect(await states()).toEqual(['done', 'running']); complete(servers[1]); await pass; expect(await states()).toEqual(['done', 'done']); expect(runningAtSubmission).toEqual([1, 1]); }); it('runs two removals concurrently, holds slots through deletion, and coalesces overlapping passes', async () => { await queue(); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 }); const { processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); const pass = track(processDestructions()); await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise)); const overlapping = track(processDestructions()); expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2); expect(await states()).toEqual(['running', 'running', 'waiting', 'waiting']); commands.get(servers[0])!.destroy.response.resolve(); await commands.get(servers[0])!.delete.started.promise; expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2); commands.get(servers[0])!.delete.response.resolve(); await commands.get(servers[2])!.destroy.started.promise; expect(await states()).toEqual(['done', 'running', 'running', 'waiting']); complete(servers[1]); await commands.get(servers[3])!.destroy.started.promise; complete(servers[2]); complete(servers[3]); await Promise.all([pass, overlapping]); expect(await states()).toEqual(['done', 'done', 'done', 'done']); expect(vi.mocked(komodoDestroyDeployment).mock.calls.map(([server]) => server)).toEqual(servers); expect(komodoDeleteDeployment).toHaveBeenCalledTimes(4); expect(Math.max(...runningAtSubmission)).toBe(2); }); it('lets active removals finish after lowering the limit and applies it to new admissions', async () => { await queue(); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 }); const { processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); const pass = track(processDestructions()); await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise)); saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 1 }); complete(servers[0]); await vi.waitFor(async () => expect(await states()).toEqual(['done', 'running', 'waiting', 'waiting'])); expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2); complete(servers[1]); await commands.get(servers[2])!.destroy.started.promise; expect(komodoDestroyDeployment).toHaveBeenCalledTimes(3); expect(await states()).toEqual(['done', 'done', 'running', 'waiting']); complete(servers[2]); await commands.get(servers[3])!.destroy.started.promise; complete(servers[3]); await pass; expect(runningAtSubmission.slice(2)).toEqual([1, 1]); }); it.each(['rejected', 'thrown'] as const)('releases capacity after a %s teardown failure', async failure => { await queue(3); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 }); const { processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); if (failure === 'rejected') { vi.mocked(komodoDestroyDeployment).mockImplementationOnce(async () => { commands.get(servers[0])!.destroy.started.resolve(); await commands.get(servers[0])!.destroy.response.promise; return { ok: false, kind: 'http-error', message: 'Teardown rejected' }; }); } const pass = track(processDestructions()); await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise)); if (failure === 'thrown') commands.get(servers[0])!.destroy.response.reject(new Error('Connection lost')); else commands.get(servers[0])!.destroy.response.resolve(); await commands.get(servers[2])!.destroy.started.promise; expect(await states()).toEqual(['failed', 'running', 'running']); complete(servers[1]); complete(servers[2]); await pass; expect(await states()).toEqual(['failed', 'done', 'done']); expect(komodoDestroyDeployment).toHaveBeenCalledTimes(3); expect(komodoDeleteDeployment).not.toHaveBeenCalledWith(servers[0]); }); it('skips occupied and unknown servers while allowing later empty and stopped servers', async () => { await queue(); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 }); const { processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(rconGetPlayerCount).mockImplementation(async server => server === servers[0] ? 2 : server === servers[2] ? 0 : null); vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === servers[1] ? { ok: false, reason: 'Host unavailable' } : { ok: false, state: 'exited', reason: 'stopped' }); for (const server of servers) complete(server); await track(processDestructions()); expect(await states()).toEqual(['waiting', 'waiting', 'done', 'done']); expect(vi.mocked(komodoDestroyDeployment).mock.calls.map(([server]) => server)).toEqual(servers.slice(2)); expect(rconGetPlayerCount).toHaveBeenCalledTimes(4); }); });