import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { existsSync } from 'node:fs'; import { readFile, rm } from 'node:fs/promises'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { T6_ZOMBIE_MAPS } from '../../src/lib/map-aliases.js'; vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoCreateDeployment: vi.fn(async () => ({ ok: true })), komodoDeploy: vi.fn(async () => ({ ok: true })), komodoDestroyDeployment: vi.fn(async () => ({ ok: true })), komodoDeleteDeployment: vi.fn(async () => ({ ok: true })), komodoInspectHealthy: vi.fn(), komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })), komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true })) })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(async () => 0), rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'localhost', port: 27016 }, rawReply: '', status: { players: 0, map: 'zombie_cosmodrome', gametype: null, roster: [] } })) })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); let fixture: Fixture; const serverName = 't5_zombie_cosmodrome_1'; const input = { gameMode: 'zombie' as const, mapSlug: 'cosmodrome', plutoKey: 'test-key' }; beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test'); vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host'); vi.stubEnv('RCON_PASSWORD', 'test-rcon'); const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconStatusResult, rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(komodoInspectHealthy).mockReset().mockResolvedValue({ ok: true, state: 'running' }); vi.mocked(rconStatusResult).mockReset().mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 27016 }, rawReply: '', status: { players: 0, map: 'zombie_cosmodrome', gametype: null, roster: [] } }); vi.mocked(rconGetPlayerCount).mockReset().mockResolvedValue(0); }); afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); describe('queued removal cancellation', () => { it('lets cancellation win while the worker awaits a player count, and never removes the server', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); const { POST } = await import('../../src/routes/api/deployments/[id]/cancel/+server.js'); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); await createServer(input); let resolveCount!: (players: number) => void; vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; })); const queued = destroyServer(serverName); const work = processDestructions(); await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function')); expect((await POST({ params: { id: queued.deploymentId! } })).status).toBe(200); resolveCount(0); await work; expect(komodoDestroyDeployment).not.toHaveBeenCalled(); expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'cancelled', server_exists: 1, stage: 'Removal cancelled' }); expect(getDeployment(queued.deploymentId!)?.canCancelRemoval).toBe(false); expect((await POST({ params: { id: queued.deploymentId! } })).status).toBe(409); }); it('does not overwrite cancellation when the in-flight player check rejects', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { cancelRemoval, getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); await createServer(input); let rejectCount!: (error: Error) => void; vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise((_resolve, reject) => { rejectCount = reject; })); const queued = destroyServer(serverName); const work = processDestructions(); await vi.waitFor(() => expect(rejectCount).toBeTypeOf('function')); expect(cancelRemoval(queued.deploymentId!)).toBe(true); rejectCount(new Error('RCON unavailable')); await work; expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'cancelled', error: null }); }); it('refuses cancellation after an atomic removal claim and for cleanup checkpoints', async () => { const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); const { cancelRemoval, claimRemoval, canCancelRemoval, deploymentStage, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js'); const { db } = await import('../../src/lib/server/db/index.js'); await createServer(input); const queued = destroyServer(serverName); expect(claimRemoval(queued.deploymentId!)).toBe(true); expect(cancelRemoval(queued.deploymentId!)).toBe(false); expect(claimRemoval(queued.deploymentId!)).toBe(false); deploymentStage(queued.deploymentId!, REMOVING_SERVER); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(queued.deploymentId); expect(canCancelRemoval(queued.deploymentId!)).toBe(false); expect(cancelRemoval(queued.deploymentId!)).toBe(false); }); }); describe('destruction cleanup recovery', () => { it('removes T6 servers with beta test runs and their dependent evidence', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: '', status: { players: 0, map: 'zm_transit', gametype: null, roster: [] } }); const name = 't6_zombie_town_1'; expect(await createServer({ ...input, game: 't6', mapSlug: 'town' })).toMatchObject({ ok: true, name }); expect(await createServer(input)).toMatchObject({ ok: true }); const { script, version } = createScript({ name: 'test script', kind: 'script', scopeType: 'game', scopeValue: 't6', targetRelPath: 'test.gsc', content: Buffer.from('main(){}') }); const beta = db().prepare("INSERT INTO betas (script_id, script_version_id, trigger_kind, state) VALUES (?, ?, 'manual', 'running')").run(script.id, version.id).lastInsertRowid; for (const server of [name, serverName]) { db().prepare("INSERT INTO beta_test_runs (id, beta_id, server, created_at) VALUES (?, ?, ?, '2026-09-18')").run(server, beta, server); db().prepare("INSERT INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, 'event', '2026-09-18', 'health', '{}')").run(server); db().prepare('INSERT INTO beta_test_coverage (beta_id, server) VALUES (?, ?)').run(beta, server); db().prepare("INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, 'test.gsc')").run(beta, server); } const queued = destroyServer(name); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' }); await processDestructions(); expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); for (const table of ['beta_test_runs', 'beta_test_coverage', 'beta_file_changes']) { expect(db().prepare(`SELECT server FROM ${table}`).all()).toEqual([{ server: serverName }]); } expect(db().prepare('SELECT run_id FROM beta_test_evidence').all()).toEqual([{ run_id: serverName }]); expect(db().pragma('foreign_key_check')).toEqual([]); }); it('resumes failed local cleanup after restart without RCON or another Komodo deletion', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); const { POST } = await import('../../src/routes/api/deployments/[id]/resume/+server.js'); await createServer(input); db().exec("CREATE TRIGGER fail_cleanup BEFORE DELETE ON servers BEGIN SELECT RAISE(ABORT, 'FOREIGN KEY constraint failed'); END"); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' }); const { deploymentId } = destroyServer(serverName); await processDestructions(); expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'failed', stage: 'Removing server files and registry', server_exists: 1 }); expect(getDeployment(deploymentId!)?.canResumeCleanup).toBe(true); db().exec('DROP TRIGGER fail_cleanup'); closeDb(); interruptDeployments(); vi.mocked(rconGetPlayerCount).mockClear(); vi.mocked(komodoDestroyDeployment).mockClear(); vi.mocked(komodoDeleteDeployment).mockClear(); expect((await POST({ params: { id: deploymentId! } })).status).toBe(202); expect((await POST({ params: { id: deploymentId! } })).status).toBe(409); await processDestructions(); expect(rconGetPlayerCount).not.toHaveBeenCalled(); expect(komodoDestroyDeployment).not.toHaveBeenCalled(); expect(komodoDeleteDeployment).not.toHaveBeenCalled(); expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); }); it('reuses an interrupted cleanup when destruction is requested again', async () => { const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); const { deploymentStage, interruptDeployments, getDeployment, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js'); const { db } = await import('../../src/lib/server/db/index.js'); await createServer(input); const first = destroyServer(serverName); deploymentStage(first.deploymentId!, REMOVING_SERVER); db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(first.deploymentId); interruptDeployments(); expect(destroyServer(serverName).deploymentId).toBe(first.deploymentId); expect(getDeployment(first.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: REMOVING_SERVER }); }); it('refuses cleanup before confirmed deletion or after a newer creation', async () => { const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); const { beginDeployment, deploymentStage, finishDeployment, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js'); await createServer(input); const { deploymentId } = destroyServer(serverName); finishDeployment(deploymentId!, { ok: false, message: 'Komodo unavailable' }); expect(resumeCleanup(deploymentId!)).toBe(false); deploymentStage(deploymentId!, REMOVING_SERVER); beginDeployment(serverName, 'create'); expect(resumeCleanup(deploymentId!)).toBe(false); }); }); describe('House Party Protocol', () => { it('includes registered and queued servers once and reuses destruction requests', async () => { const { createServer, queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js'); expect(await createServer(input)).toMatchObject({ ok: true }); expect(queueCreateServer(input)).toMatchObject({ ok: true }); const names = housePartyTargets(); expect(names).toEqual([serverName, 't5_zombie_cosmodrome_2']); const first = queueHouseParty(names); expect(first).toMatchObject({ ok: true, deployments: [{ server: serverName }, { server: names[1] }] }); expect(queueHouseParty(names)).toEqual(first); }); it('rejects a changed fleet without queuing any destruction', async () => { const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js'); const { db } = await import('../../src/lib/server/db/index.js'); const names = housePartyTargets(); expect(queueCreateServer(input).ok).toBe(true); expect(queueHouseParty(names).ok).toBe(false); expect(db().prepare("SELECT * FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]); }); it('rolls back the entire bulk request when an insert fails', async () => { const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js'); const { db } = await import('../../src/lib/server/db/index.js'); expect(queueCreateServer(input).ok).toBe(true); expect(queueCreateServer(input).ok).toBe(true); db().exec(`CREATE TRIGGER reject_destroy BEFORE INSERT ON server_deployments WHEN NEW.action = 'destroy' AND NEW.server = 't5_zombie_cosmodrome_2' BEGIN SELECT RAISE(ABORT, 'test failure'); END`); expect(() => queueHouseParty(housePartyTargets())).toThrow('test failure'); expect(db().prepare("SELECT * FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]); }); it('waits for queued creations, then for verified emptiness, and preserves keys', async () => { const { queueCreateServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); expect(queueCreateServer(input).ok).toBe(true); expect(queueHouseParty(housePartyTargets()).ok).toBe(true); await processDestructions(); expect(komodoDestroyDeployment).not.toHaveBeenCalled(); await processCreations(); vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2); await processDestructions(); vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(null); await processDestructions(); expect(komodoDestroyDeployment).not.toHaveBeenCalled(); vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(0); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, kind: 'not-found', message: 'gone' } as never); await processDestructions(); expect(komodoDestroyDeployment).toHaveBeenCalledWith(serverName); expect(getServer(serverName)).toBeUndefined(); expect(getServerKey(serverName)).toBe('test-key'); }); }); async function promotedConfig() { const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); for (const artifact of [ { name: 'baseline', kind: 'baseline' as const, scopeType: 'game' as const, scopeValue: 't5', content: Buffer.from('set sv_voice "1"\nset sv_maprotation "map zombie_theater"\n') }, { name: 'map settings', kind: 'config' as const, scopeType: 'map' as const, scopeValue: 'zombie_cosmodrome', content: Buffer.from('set sv_maprotation "map zombie_moon"\nset g_inactivity "0"\n') }, { name: 'other map', kind: 'config' as const, scopeType: 'map' as const, scopeValue: 'zombie_theater', content: Buffer.from('set sv_voice "0"\n') } ]) { const { script } = createScript(artifact); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?') .run(getLatestVersion(script.id)!.id, script.id); } } describe('T6 Zombies provisioning', () => { it('keeps T6 storage and config overrides separate from T5 settings', async () => { vi.stubEnv('T6_SERVERS_ROOT', join(fixture.root, 'custom-t6')); vi.stubEnv('T6_CONFIG_TARGET_FILE', 'custom_zm.cfg'); vi.stubEnv('T6_REL_DEST_SCRIPT', 'custom/scripts/zm'); vi.stubEnv('PLUTAINER_T6_GAMEFILES_VOLUME', 'bo2-files'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); const baseline = createScript({ name: 'T6 baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't6', content: Buffer.from('sv_voice 1\n') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(baseline.version.id, baseline.script.id); const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); const { destOf, configFileOf } = await import('../../src/lib/server/config.js'); const created = await createServer({ ...input, game: 't6', mapSlug: 'town' }); if (!created.ok) throw new Error(created.message); const spec = vi.mocked(komodoCreateDeployment).mock.calls[0][0]; expect(spec.volumes).toContain('bo2-files:/home/plutainer/gamefiles:ro'); expect(spec.environment).toContainEqual({ variable: 'PLUTAINER_CONFIG_FILE', value: 'custom_zm.cfg' }); expect(existsSync(join(fixture.root, 'custom-t6', created.name, 'configs/custom_zm.cfg'))).toBe(true); expect(destOf(created.name)).toBe(`${join(fixture.root, 'custom-t6')}/${created.name}/custom/scripts/zm`); expect(configFileOf(serverName)).toBe('dedicated_sp.cfg'); expect(destOf(serverName)).toBe(`${fixture.t5}/${serverName}/runtime/plutonium/storage/t5/raw/scripts/sp`); }); it.each(T6_ZOMBIE_MAPS)('provisions $friendly with the T6 mount, config, and ruleset', async (map) => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); const name = `t6_zombie_${map.slug}_1`; expect(await createServer({ ...input, game: 't6', mapSlug: map.slug })).toMatchObject({ ok: true, name }); const spec = vi.mocked(komodoCreateDeployment).mock.calls[0][0]; const env = Object.fromEntries(spec.environment.map(({ variable, value }) => [variable, value])); expect(env).toMatchObject({ PLUTAINER_GAME: 't6zm', PLUTAINER_CONFIG_FILE: 'dedicated_zm.cfg', PLUTAINER_RCON_PASSWORD: 'test-rcon', PLUTAINER_MAP_ROTATE: 'false', PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${name} +map_rotate` }); expect(env.PLUTAINER_SKIP_SEED).toBeUndefined(); expect(spec.volumes).toEqual([`${fixture.t6}/${name}:/home/plutainer/app`, 't6_game_files:/home/plutainer/gamefiles:ro']); expect(getServer(name)).toMatchObject({ game: 't6', game_id: 't6zm', map: map.canonical, map_slug: map.slug, config_file: 'dedicated_zm.cfg' }); expect(await readFile(join(fixture.t6, name, 'configs/electric_cherry_map.cfg'), 'utf8')) .toContain(`set sv_maprotation "execgts ${map.gameSettings} map ${map.canonical}"`); expect(existsSync(join(fixture.t5, name))).toBe(false); expect(existsSync(join(fixture.t6, name, 'configs/dedicated_zm.cfg'))).toBe(false); }); it('persists T6 queued work, reuses its key, and allocates ports across both games', async () => { const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); const { saveServerKey, validServerId } = await import('../../src/lib/server/fleet/server-keys.js'); const { closeDb } = await import('../../src/lib/server/db/index.js'); const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); saveServerKey('t6_zombie_town_1', 't6-key'); expect(validServerId('t6_zombie_town_20')).toBe(true); expect(validServerId('t5_zombie_town_1')).toBe(false); expect(validServerId('t6_zombie_theater_1')).toBe(false); const t5 = queueCreateServer(input); const t6 = queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town' }); if (!t5.ok || !t6.ok) throw new Error('queue failed'); expect(t6.port).not.toBe(t5.port); closeDb(); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); vi.mocked(rconStatusResult).mockImplementation(async (server) => ({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: '', status: { players: 0, map: server.startsWith('t6_') ? 'zm_transit' : 'zombie_cosmodrome', gametype: null, roster: [] } })); await processCreations(); expect(getDeployment(t5.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' }); expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'queued' }); await processCreations(); expect(getDeployment(t5.deploymentId)?.deployment).toMatchObject({ state: 'done' }); expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' }); await processCreations(); expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'done' }); const content = await readFile(join(fixture.t6, t6.name, 'configs/electric_cherry_map.cfg'), 'utf8'); expect(content).toContain('execgts zm_standard_town.cfg map zm_transit'); }); it('installs only matching promoted artifacts into T6 paths and removes only its directory on destruction', async () => { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); for (const game of ['t5', 't6']) { for (const kind of ['script', 'baseline'] as const) { const { script, version } = createScript({ name: `${game}-${kind}`, kind, scopeType: 'game', scopeValue: game, targetRelPath: kind === 'script' ? `${game}.gsc` : undefined, content: Buffer.from(kind === 'script' ? 'main(){}' : 'sv_voice 1\nsv_maprotation "map wrong"\n') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); } } const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js'); await createServer(input); const t6 = await createServer({ ...input, game: 't6', mapSlug: 'tomb' }); if (!t6.ok) throw new Error(t6.message); expect(existsSync(join(fixture.t6, t6.name, 'runtime/plutonium/storage/t6/scripts/zm/t6.gsc'))).toBe(true); expect(existsSync(join(fixture.t6, t6.name, 'runtime/plutonium/storage/t6/scripts/zm/t5.gsc'))).toBe(false); expect(await readFile(join(fixture.t6, t6.name, 'configs/dedicated_zm.cfg'), 'utf8')) .toContain('execgts zm_classic_tomb.cfg map zm_tomb'); expect(await scanServerDrift(t6.name)).toEqual([]); expect(destroyServer(t6.name).ok).toBe(true); const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }); await processDestructions(); expect(existsSync(join(fixture.t6, t6.name))).toBe(false); expect(existsSync(join(fixture.t5, serverName))).toBe(true); expect(getServerKey(t6.name)).toBe('test-key'); }); it('validates game/map combinations at the API boundary', async () => { const { POST } = await import('../../src/routes/api/servers/+server.js'); for (const game of ['t7', null, 6]) { const res = await POST({ request: new Request('http://localhost/api/servers', { method: 'POST', body: JSON.stringify({ ...input, game }) }) }); expect(res.status).toBe(400); } const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); expect(queueCreateServer({ ...input, game: 't6' })).toMatchObject({ ok: false, kind: 'unknown-map' }); }); }); describe('server creation map selection', () => { it('returns a queued creation before provisioning and reserves its name and port', async () => { const { POST } = await import('../../src/routes/api/servers/+server.js'); const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { closeDb } = await import('../../src/lib/server/db/index.js'); const response = await POST({ request: new Request('http://localhost/api/servers', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(input) }) }); expect(response.status).toBe(202); const body = await response.json(); expect(body).toMatchObject({ name: serverName, deploymentId: expect.any(String) }); expect(komodoCreateDeployment).not.toHaveBeenCalled(); expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'queued', expected_map: 'zombie_cosmodrome' }); expect(queueCreateServer({ ...input, expectedName: serverName })).toMatchObject({ ok: false, kind: 'no-slot' }); const second = queueCreateServer(input); expect(second).toMatchObject({ ok: true, name: 't5_zombie_cosmodrome_2' }); if (second.ok) expect(second.port).not.toBe(body.port); closeDb(); interruptDeployments(); expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'queued' }); await Promise.all([processCreations(), processCreations()]); expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null }); if (second.ok) expect(getDeployment(second.deploymentId)?.deployment).toMatchObject({ state: 'queued' }); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }); await processCreations(); expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); }); it('completes creation only when Komodo, RCON, and the selected map all pass', async () => { const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); const created = await createServer(input); if (!created.ok) throw new Error('fixture creation failed'); const id = created.deploymentId; expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null }); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }); await processCreations(); expect(rconStatusResult).not.toHaveBeenCalled(); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'not ready' }); await processCreations(); expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', stage: 'Waiting for RCON or fresh telemetry health: timeout' }); for (const map of [null, 'zombie_moon']) { vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'localhost', port: created.port }, rawReply: '', status: { players: 0, map, gametype: null, roster: [] } }); await processCreations(); expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null, stage: expect.stringContaining('expected zombie_cosmodrome') }); } interruptDeployments(); expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health' }); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); await processCreations(); expect(getDeployment(id)?.deployment).toMatchObject({ state: 'done', finished_at: expect.any(String), stage: 'Verified Komodo healthy, RCON healthy, and map zombie_cosmodrome' }); }); it('returns a deployment URL identifier immediately and serves its lifecycle detail', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js'); const { GET } = await import('../../src/routes/api/deployments/[id]/+server.js'); const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); await createServer(input); const response = await DELETE({ params: { name: serverName } }); expect(response.status).toBe(202); const body = await response.json(); expect(body).toMatchObject({ ok: true, deploymentId: expect.any(String) }); expect(komodoDestroyDeployment).not.toHaveBeenCalled(); const detail = await (await GET({ params: { id: body.deploymentId } })).json(); expect(detail.deployment).toMatchObject({ server: serverName, action: 'destroy', state: 'waiting' }); expect(detail.events).toHaveLength(2); const missing = await DELETE({ params: { name: 'missing-server' } }); expect(missing.status).toBe(404); }); it('queues destruction, waits for live zero players, and preserves an ordered timeline', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { closeDb } = await import('../../src/lib/server/db/index.js'); await createServer(input); const queued = destroyServer(serverName); expect(queued).toMatchObject({ ok: true, deploymentId: expect.any(String) }); expect(destroyServer(serverName).deploymentId).toBe(queued.deploymentId); expect(komodoDestroyDeployment).not.toHaveBeenCalled(); vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(3).mockResolvedValueOnce(null).mockRejectedValueOnce(new Error('RCON unavailable')); for (let i = 0; i < 3; i++) { await processDestructions(); expect(komodoDestroyDeployment).not.toHaveBeenCalled(); expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting' }); } closeDb(); interruptDeployments(); expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting' }); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }); await Promise.all([processDestructions(), processDestructions()]); expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1); const detail = getDeployment(queued.deploymentId!); expect(detail?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); const messages = (detail?.events as { message: string }[]).map(e => e.message); expect(messages).toEqual(expect.arrayContaining(['Server destruction requested', 'Waiting for server to empty: 3 player(s) online', 'Server is empty (0 players verified)', 'Stopping and removing container', 'Completed'])); expect(messages.indexOf('Server is empty (0 players verified)')).toBeLessThan(messages.indexOf('Stopping and removing container')); }); it('preserves files and registry when Komodo rejects destruction and exposes failure details', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); await createServer(input); vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'denied' }); const queued = destroyServer(serverName); await processDestructions(); expect(komodoDeleteDeployment).not.toHaveBeenCalled(); expect(existsSync(join(fixture.t5, serverName))).toBe(true); expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'failed', server_exists: 1, error: 'DestroyDeployment failed: denied' }); }); it('records in-flight creation and retains both operations after destruction', async () => { const { createServer, destroyServer, processDestructions, processCreations } = await import('../../src/lib/server/fleet/create.js'); const { listDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { komodoDeploy, komodoInspectHealthy, komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoDeploy).mockImplementationOnce(async () => { expect(listDeployments()).toContainEqual(expect.objectContaining({ server: serverName, action: 'create', state: 'running', stage: 'Starting container' })); return { ok: true }; }); expect((await createServer(input)).ok).toBe(true); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); await processCreations(); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }); vi.mocked(komodoDestroyDeployment).mockImplementationOnce(async () => { expect(listDeployments()).toContainEqual(expect.objectContaining({ action: 'destroy', state: 'running' })); return { ok: true }; }); expect((await destroyServer(serverName)).ok).toBe(true); await processDestructions(); expect(listDeployments()).toEqual(expect.arrayContaining([ expect.objectContaining({ server: serverName, action: 'create', state: 'done', server_exists: 0 }), expect.objectContaining({ server: serverName, action: 'destroy', state: 'done', server_exists: 0 }) ])); }); it('records failed and unexpectedly interrupted creation attempts', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { listDeployments, beginDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'test failure' }); expect((await createServer(input)).ok).toBe(false); expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'failed', error: 'Deploy: test failure' })); vi.mocked(komodoDeploy).mockRejectedValueOnce(new Error('connection lost')); await expect(createServer(input)).rejects.toThrow('connection lost'); expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'awaiting-health', stage: 'Verifying existing server after interrupted startup', finished_at: null })); const id = beginDeployment('interrupted-server', 'destroy'); interruptDeployments(); expect(listDeployments()).toContainEqual(expect.objectContaining({ id, state: 'interrupted', finished_at: expect.any(String) })); }); it('installs only promoted scope-matching scripts and plugins before first boot', async () => { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { destOfKind } = await import('../../src/lib/server/config.js'); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js'); const artifacts = [ { kind: 'script' as const, target: 'zm_spawn_fix.gsc', scopeType: 'game' as const, scopeValue: 't5', promoted: true, expected: true }, { kind: 'plugin' as const, target: 'plugin.dll', scopeType: 'map' as const, scopeValue: 'zombie_cosmodrome', promoted: true, expected: true }, { kind: 'script' as const, target: 'other.gsc', scopeType: 'map' as const, scopeValue: 'zombie_theater', promoted: true, expected: false }, { kind: 'script' as const, target: 'draft.gsc', scopeType: 'game' as const, scopeValue: 't5', promoted: false, expected: false } ].map((a) => { const { script, version } = createScript({ name: a.target, kind: a.kind, targetRelPath: a.target, scopeType: a.scopeType, scopeValue: a.scopeValue, content: Buffer.from(a.target) }); if (a.promoted) db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); return { ...a, version }; }); vi.mocked(komodoDeploy).mockImplementationOnce(async () => { for (const a of artifacts) { const path = join(destOfKind(serverName, a.kind), a.target); expect(existsSync(path)).toBe(a.expected); if (a.expected) { expect(await readFile(path)).toEqual(a.version.content); expect(getServerScript(serverName, a.target)).toMatchObject({ script_version_id: a.version.id, source: 'create' }); } } return { ok: true }; }); expect(await createServer(input)).toMatchObject({ ok: true }); expect(await scanServerDrift(serverName)).toEqual([]); }); it('keeps Plutainer defaults without a baseline and overrides the map before rotation', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); const result = await createServer(input); expect(result).toMatchObject({ ok: true, name: serverName }); expect(existsSync(join(fixture.t5, serverName, 'configs', 'dedicated_sp.cfg'))).toBe(false); const spec = vi.mocked(komodoCreateDeployment).mock.calls[0][0]; const env = Object.fromEntries(spec.environment.map(({ variable, value }) => [variable, value])); expect(env.PLUTAINER_MAP_ROTATE).toBe('false'); expect(env.PLUTAINER_EXTRA_ARGS).toBe(`+exec electric_cherry_map.cfg +set tws_server_id ${serverName} +map_rotate`); expect(await readFile(join(fixture.t5, serverName, 'configs', 'electric_cherry_map.cfg'), 'utf8')) .toContain('set sv_maprotation "map zombie_cosmodrome"'); expect(env.PLUTAINER_CONFIG_FILE).toBe('dedicated_sp.cfg'); expect(env.PLUTAINER_SKIP_SEED).toBeUndefined(); expect(komodoDeploy).toHaveBeenCalledWith(serverName); expect(getServer(serverName)?.map).toBe('zombie_cosmodrome'); const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); expect(await scanServerDrift(serverName)).toEqual([]); const mapPath = join(fixture.t5, serverName, 'configs', 'electric_cherry_map.cfg'); await rm(mapPath); expect((await runDriftDeployment({ servers: [serverName] })).totalRestored).toBe(1); expect(await readFile(mapPath, 'utf8')).toContain('set sv_maprotation "map zombie_cosmodrome"'); expect(await scanServerDrift(serverName)).toEqual([]); }); it('writes promoted settings and selected map before first boot and tracks the file', async () => { await promotedConfig(); const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js'); vi.mocked(komodoDeploy).mockImplementationOnce(async () => { const config = await readFile(join(fixture.t5, serverName, 'configs', 'dedicated_sp.cfg'), 'utf8'); expect(config).toContain('set sv_maprotation "map zombie_cosmodrome"'); expect(config).toContain('set sv_voice "1"'); expect(config).toContain('set g_inactivity "0"'); expect(config).not.toContain('map zombie_moon'); expect(getServerScript(serverName, 'dedicated_sp.cfg')?.source).toBe('create'); return { ok: true }; }); expect(await createServer(input)).toMatchObject({ ok: true }); expect(await scanServerDrift(serverName)).toEqual([]); }); it('removes the generated config and inventory if deployment fails', async () => { await promotedConfig(); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { script, version } = createScript({ name: 'spawn fix', kind: 'script', scopeType: 'game', scopeValue: 't5', targetRelPath: 'zm_spawn_fix.gsc', content: Buffer.from('main(){}') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'test failure' }); expect(await createServer(input)).toMatchObject({ ok: false, kind: 'komodo' }); expect(getServer(serverName)).toBeUndefined(); expect(getServerScript(serverName, 'dedicated_sp.cfg')).toBeUndefined(); expect(getServerScript(serverName, 'zm_spawn_fix.gsc')).toBeUndefined(); expect(existsSync(join(fixture.t5, serverName))).toBe(false); }); it('rejects unknown maps before creating a deployment', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); expect(await createServer({ ...input, mapSlug: 'unknown' })).toMatchObject({ ok: false, kind: 'unknown-map' }); expect(komodoCreateDeployment).not.toHaveBeenCalled(); }); }); describe('persistent server keys', () => { it('saves a key, retains it through destruction, and reuses it for the exact lobby', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { getServerKey, listServerKeys } = await import('../../src/lib/server/fleet/server-keys.js'); const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const name = 't5_zombie_cosmodrome_2'; expect(await createServer(input)).toMatchObject({ ok: true, name: serverName }); expect(await createServer(input)).toMatchObject({ ok: true, name }); expect(getServerKey(name)).toBe('test-key'); expect(await destroyServer(name)).toMatchObject({ ok: true }); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' }); await processDestructions(); expect(listServerKeys()).toContainEqual(expect.objectContaining({ server_id: name, deployed: 0 })); expect(JSON.stringify(listServerKeys())).not.toContain('test-key'); expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', expectedName: name })).toMatchObject({ ok: true, name }); const spec = vi.mocked(komodoCreateDeployment).mock.calls.at(-1)![0]; expect(spec.environment).toContainEqual({ variable: 'PLUTO_SERVER_KEY', value: 'test-key' }); expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome' })).toMatchObject({ ok: false, kind: 'bad-request' }); }); it('updates and removes keys independently of deployments and survives reopening the database', async () => { const { saveServerKey, getServerKey, removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); const { closeDb } = await import('../../src/lib/server/db/index.js'); saveServerKey(serverName, ' first-key '); closeDb(); expect(getServerKey(serverName)).toBe('first-key'); saveServerKey(serverName, 'replacement'); expect(getServerKey(serverName)).toBe('replacement'); expect(removeServerKey(serverName)).toBe(true); expect(getServerKey(serverName)).toBeUndefined(); expect(() => saveServerKey('t5_zombie_unknown_1', 'key')).toThrow(); expect(() => saveServerKey(serverName, ' ')).toThrow(); expect(() => saveServerKey(serverName, 'two words')).toThrow(); }); it('fills the lowest gap and rejects stale previews before saving a key or deploying', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); expect(await createServer(input)).toMatchObject({ ok: true, name: serverName }); expect(await createServer(input)).toMatchObject({ ok: true, name: 't5_zombie_cosmodrome_2' }); await destroyServer(serverName); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' }); await processDestructions(); expect(await createServer({ ...input, plutoKey: 'wrong-key', expectedName: 't5_zombie_cosmodrome_3' })) .toMatchObject({ ok: false, kind: 'no-slot' }); expect(getServerKey(serverName)).toBe('test-key'); expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', expectedName: serverName })) .toMatchObject({ ok: true, name: serverName }); }); });