import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { existsSync } from 'node:fs'; import { readFile, rm } from 'node:fs/promises'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoCreateDeployment: vi.fn(async () => ({ ok: true })), komodoDeploy: vi.fn(async () => ({ ok: true })), komodoDestroyDeployment: vi.fn(async () => ({ ok: true })), komodoDeleteDeployment: vi.fn(async () => ({ ok: true })), komodoInspectHealthy: vi.fn(), komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })), komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true })) })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(async () => 0), rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'localhost', port: 27016 }, rawReply: '', status: { players: 0, map: 'zombie_cosmodrome', gametype: null, roster: [] } })) })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); let fixture: Fixture; const serverName = 't5_zombie_cosmodrome_1'; const input = { gameMode: 'zombie' as const, mapSlug: 'cosmodrome', plutoKey: 'test-key' }; beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test'); vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host'); vi.stubEnv('RCON_PASSWORD', 'test-rcon'); }); afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); async function promotedConfig() { const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); for (const artifact of [ { name: 'baseline', kind: 'baseline' as const, scopeType: 'game' as const, scopeValue: 't5', content: Buffer.from('set sv_voice "1"\nset sv_maprotation "map zombie_theater"\n') }, { name: 'map settings', kind: 'config' as const, scopeType: 'map' as const, scopeValue: 'zombie_cosmodrome', content: Buffer.from('set sv_maprotation "map zombie_moon"\nset g_inactivity "0"\n') }, { name: 'other map', kind: 'config' as const, scopeType: 'map' as const, scopeValue: 'zombie_theater', content: Buffer.from('set sv_voice "0"\n') } ]) { const { script } = createScript(artifact); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?') .run(getLatestVersion(script.id)!.id, script.id); } } describe('server creation map selection', () => { it('returns a queued creation before provisioning and reserves its name and port', async () => { const { POST } = await import('../../src/routes/api/servers/+server.js'); const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { closeDb } = await import('../../src/lib/server/db/index.js'); const response = await POST({ request: new Request('http://localhost/api/servers', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(input) }) }); expect(response.status).toBe(202); const body = await response.json(); expect(body).toMatchObject({ name: serverName, deploymentId: expect.any(String) }); expect(komodoCreateDeployment).not.toHaveBeenCalled(); expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'queued', expected_map: 'zombie_cosmodrome' }); expect(queueCreateServer({ ...input, expectedName: serverName })).toMatchObject({ ok: false, kind: 'no-slot' }); const second = queueCreateServer(input); expect(second).toMatchObject({ ok: true, name: 't5_zombie_cosmodrome_2' }); if (second.ok) expect(second.port).not.toBe(body.port); closeDb(); interruptDeployments(); expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'queued' }); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }).mockResolvedValueOnce({ ok: false, reason: 'starting' }); await Promise.all([processCreations(), processCreations()]); expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null }); }); it('completes creation only when Komodo, RCON, and the selected map all pass', async () => { const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); const created = await createServer(input); if (!created.ok) throw new Error('fixture creation failed'); const id = created.deploymentId; expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null }); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }); await processCreations(); expect(rconStatusResult).not.toHaveBeenCalled(); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'not ready' }); await processCreations(); expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', stage: 'Waiting for RCON health: timeout' }); for (const map of [null, 'zombie_moon']) { vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'localhost', port: created.port }, rawReply: '', status: { players: 0, map, gametype: null, roster: [] } }); await processCreations(); expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null, stage: expect.stringContaining('expected zombie_cosmodrome') }); } interruptDeployments(); expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health' }); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); await processCreations(); expect(getDeployment(id)?.deployment).toMatchObject({ state: 'done', finished_at: expect.any(String), stage: 'Verified Komodo healthy, RCON healthy, and map zombie_cosmodrome' }); }); it('returns a deployment URL identifier immediately and serves its lifecycle detail', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js'); const { GET } = await import('../../src/routes/api/deployments/[id]/+server.js'); const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); await createServer(input); const response = await DELETE({ params: { name: serverName } }); expect(response.status).toBe(202); const body = await response.json(); expect(body).toMatchObject({ ok: true, deploymentId: expect.any(String) }); expect(komodoDestroyDeployment).not.toHaveBeenCalled(); const detail = await (await GET({ params: { id: body.deploymentId } })).json(); expect(detail.deployment).toMatchObject({ server: serverName, action: 'destroy', state: 'waiting' }); expect(detail.events).toHaveLength(2); const missing = await DELETE({ params: { name: 'missing-server' } }); expect(missing.status).toBe(404); }); it('queues destruction, waits for live zero players, and preserves an ordered timeline', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { getDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { closeDb } = await import('../../src/lib/server/db/index.js'); await createServer(input); const queued = destroyServer(serverName); expect(queued).toMatchObject({ ok: true, deploymentId: expect.any(String) }); expect(destroyServer(serverName).deploymentId).toBe(queued.deploymentId); expect(komodoDestroyDeployment).not.toHaveBeenCalled(); vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(3).mockResolvedValueOnce(null).mockRejectedValueOnce(new Error('RCON unavailable')); for (let i = 0; i < 3; i++) { await processDestructions(); expect(komodoDestroyDeployment).not.toHaveBeenCalled(); expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting' }); } closeDb(); interruptDeployments(); expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting' }); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }); await Promise.all([processDestructions(), processDestructions()]); expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1); const detail = getDeployment(queued.deploymentId!); expect(detail?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); const messages = (detail?.events as { message: string }[]).map(e => e.message); expect(messages).toEqual(expect.arrayContaining(['Server destruction requested', 'Waiting for server to empty: 3 player(s) online', 'Server is empty (0 players verified)', 'Stopping and removing container', 'Completed'])); expect(messages.indexOf('Server is empty (0 players verified)')).toBeLessThan(messages.indexOf('Stopping and removing container')); }); it('preserves files and registry when Komodo rejects destruction and exposes failure details', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); await createServer(input); vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'denied' }); const queued = destroyServer(serverName); await processDestructions(); expect(komodoDeleteDeployment).not.toHaveBeenCalled(); expect(existsSync(join(fixture.t5, serverName))).toBe(true); expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'failed', server_exists: 1, error: 'DestroyDeployment failed: denied' }); }); it('records in-flight creation and retains both operations after destruction', async () => { const { createServer, destroyServer, processDestructions, processCreations } = await import('../../src/lib/server/fleet/create.js'); const { listDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { komodoDeploy, komodoInspectHealthy, komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoDeploy).mockImplementationOnce(async () => { expect(listDeployments()).toContainEqual(expect.objectContaining({ server: serverName, action: 'create', state: 'running', stage: 'Starting container' })); return { ok: true }; }); expect((await createServer(input)).ok).toBe(true); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); await processCreations(); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }); vi.mocked(komodoDestroyDeployment).mockImplementationOnce(async () => { expect(listDeployments()).toContainEqual(expect.objectContaining({ action: 'destroy', state: 'running' })); return { ok: true }; }); expect((await destroyServer(serverName)).ok).toBe(true); await processDestructions(); expect(listDeployments()).toEqual(expect.arrayContaining([ expect.objectContaining({ server: serverName, action: 'create', state: 'done', server_exists: 0 }), expect.objectContaining({ server: serverName, action: 'destroy', state: 'done', server_exists: 0 }) ])); }); it('records failed and unexpectedly interrupted creation attempts', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { listDeployments, beginDeployment, interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js'); const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'test failure' }); expect((await createServer(input)).ok).toBe(false); expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'failed', error: 'Deploy: test failure' })); vi.mocked(komodoDeploy).mockRejectedValueOnce(new Error('connection lost')); await expect(createServer(input)).rejects.toThrow('connection lost'); expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'failed', error: 'connection lost' })); const id = beginDeployment('interrupted-server', 'destroy'); interruptDeployments(); expect(listDeployments()).toContainEqual(expect.objectContaining({ id, state: 'interrupted', finished_at: expect.any(String) })); }); it('installs only promoted scope-matching scripts and plugins before first boot', async () => { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { destOfKind } = await import('../../src/lib/server/config.js'); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js'); const artifacts = [ { kind: 'script' as const, target: 'zm_spawn_fix.gsc', scopeType: 'game' as const, scopeValue: 't5', promoted: true, expected: true }, { kind: 'plugin' as const, target: 'plugin.dll', scopeType: 'map' as const, scopeValue: 'zombie_cosmodrome', promoted: true, expected: true }, { kind: 'script' as const, target: 'other.gsc', scopeType: 'map' as const, scopeValue: 'zombie_theater', promoted: true, expected: false }, { kind: 'script' as const, target: 'draft.gsc', scopeType: 'game' as const, scopeValue: 't5', promoted: false, expected: false } ].map((a) => { const { script, version } = createScript({ name: a.target, kind: a.kind, targetRelPath: a.target, scopeType: a.scopeType, scopeValue: a.scopeValue, content: Buffer.from(a.target) }); if (a.promoted) db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); return { ...a, version }; }); vi.mocked(komodoDeploy).mockImplementationOnce(async () => { for (const a of artifacts) { const path = join(destOfKind(serverName, a.kind), a.target); expect(existsSync(path)).toBe(a.expected); if (a.expected) { expect(await readFile(path)).toEqual(a.version.content); expect(getServerScript(serverName, a.target)).toMatchObject({ script_version_id: a.version.id, source: 'create' }); } } return { ok: true }; }); expect(await createServer(input)).toMatchObject({ ok: true }); expect(await scanServerDrift(serverName)).toEqual([]); }); it('keeps Plutainer defaults without a baseline and overrides the map before rotation', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); const result = await createServer(input); expect(result).toMatchObject({ ok: true, name: serverName }); expect(existsSync(join(fixture.t5, serverName, 'configs', 'dedicated_sp.cfg'))).toBe(false); const spec = vi.mocked(komodoCreateDeployment).mock.calls[0][0]; const env = Object.fromEntries(spec.environment.map(({ variable, value }) => [variable, value])); expect(env.PLUTAINER_MAP_ROTATE).toBe('false'); expect(env.PLUTAINER_EXTRA_ARGS).toBe(`+exec electric_cherry_map.cfg +set tws_server_id ${serverName} +map_rotate`); expect(await readFile(join(fixture.t5, serverName, 'configs', 'electric_cherry_map.cfg'), 'utf8')) .toContain('set sv_maprotation "map zombie_cosmodrome"'); expect(env.PLUTAINER_CONFIG_FILE).toBe('dedicated_sp.cfg'); expect(env.PLUTAINER_SKIP_SEED).toBeUndefined(); expect(komodoDeploy).toHaveBeenCalledWith(serverName); expect(getServer(serverName)?.map).toBe('zombie_cosmodrome'); const { scanServerDrift, runDriftDeployment } = await import('../../src/lib/server/fleet/drift.js'); expect(await scanServerDrift(serverName)).toEqual([]); const mapPath = join(fixture.t5, serverName, 'configs', 'electric_cherry_map.cfg'); await rm(mapPath); expect((await runDriftDeployment({ servers: [serverName] })).totalRestored).toBe(1); expect(await readFile(mapPath, 'utf8')).toContain('set sv_maprotation "map zombie_cosmodrome"'); expect(await scanServerDrift(serverName)).toEqual([]); }); it('writes promoted settings and selected map before first boot and tracks the file', async () => { await promotedConfig(); const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); const { scanServerDrift } = await import('../../src/lib/server/fleet/drift.js'); vi.mocked(komodoDeploy).mockImplementationOnce(async () => { const config = await readFile(join(fixture.t5, serverName, 'configs', 'dedicated_sp.cfg'), 'utf8'); expect(config).toContain('set sv_maprotation "map zombie_cosmodrome"'); expect(config).toContain('set sv_voice "1"'); expect(config).toContain('set g_inactivity "0"'); expect(config).not.toContain('map zombie_moon'); expect(getServerScript(serverName, 'dedicated_sp.cfg')?.source).toBe('create'); return { ok: true }; }); expect(await createServer(input)).toMatchObject({ ok: true }); expect(await scanServerDrift(serverName)).toEqual([]); }); it('removes the generated config and inventory if deployment fails', async () => { await promotedConfig(); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { script, version } = createScript({ name: 'spawn fix', kind: 'script', scopeType: 'game', scopeValue: 't5', targetRelPath: 'zm_spawn_fix.gsc', content: Buffer.from('main(){}') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'test failure' }); expect(await createServer(input)).toMatchObject({ ok: false, kind: 'komodo' }); expect(getServer(serverName)).toBeUndefined(); expect(getServerScript(serverName, 'dedicated_sp.cfg')).toBeUndefined(); expect(getServerScript(serverName, 'zm_spawn_fix.gsc')).toBeUndefined(); expect(existsSync(join(fixture.t5, serverName))).toBe(false); }); it('rejects unknown maps before creating a deployment', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); expect(await createServer({ ...input, mapSlug: 'unknown' })).toMatchObject({ ok: false, kind: 'unknown-map' }); expect(komodoCreateDeployment).not.toHaveBeenCalled(); }); }); describe('persistent server keys', () => { it('saves a key, retains it through destruction, and reuses it for the exact lobby', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { getServerKey, listServerKeys } = await import('../../src/lib/server/fleet/server-keys.js'); const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, kind: 'not-found', message: 'gone' } as never); const name = 't5_zombie_cosmodrome_2'; expect(await createServer(input)).toMatchObject({ ok: true, name: serverName }); expect(await createServer(input)).toMatchObject({ ok: true, name }); expect(getServerKey(name)).toBe('test-key'); expect(await destroyServer(name)).toMatchObject({ ok: true }); await processDestructions(); expect(listServerKeys()).toContainEqual(expect.objectContaining({ server_id: name, deployed: 0 })); expect(JSON.stringify(listServerKeys())).not.toContain('test-key'); expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', expectedName: name })).toMatchObject({ ok: true, name }); const spec = vi.mocked(komodoCreateDeployment).mock.calls.at(-1)![0]; expect(spec.environment).toContainEqual({ variable: 'PLUTO_SERVER_KEY', value: 'test-key' }); expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome' })).toMatchObject({ ok: false, kind: 'bad-request' }); }); it('updates and removes keys independently of deployments and survives reopening the database', async () => { const { saveServerKey, getServerKey, removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); const { closeDb } = await import('../../src/lib/server/db/index.js'); saveServerKey(serverName, ' first-key '); closeDb(); expect(getServerKey(serverName)).toBe('first-key'); saveServerKey(serverName, 'replacement'); expect(getServerKey(serverName)).toBe('replacement'); expect(removeServerKey(serverName)).toBe(true); expect(getServerKey(serverName)).toBeUndefined(); expect(() => saveServerKey('t5_zombie_unknown_1', 'key')).toThrow(); expect(() => saveServerKey(serverName, ' ')).toThrow(); expect(() => saveServerKey(serverName, 'two words')).toThrow(); }); it('fills the lowest gap and rejects stale previews before saving a key or deploying', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false } as never); expect(await createServer(input)).toMatchObject({ ok: true, name: serverName }); expect(await createServer(input)).toMatchObject({ ok: true, name: 't5_zombie_cosmodrome_2' }); await destroyServer(serverName); await processDestructions(); expect(await createServer({ ...input, plutoKey: 'wrong-key', expectedName: 't5_zombie_cosmodrome_3' })) .toMatchObject({ ok: false, kind: 'no-slot' }); expect(getServerKey(serverName)).toBe('test-key'); expect(komodoCreateDeployment).toHaveBeenCalledTimes(2); expect(await createServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', expectedName: serverName })) .toMatchObject({ ok: true, name: serverName }); }); });