import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import type { CollectionPreview } from '../../src/lib/deployment-plan.js'; vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoCreateDeployment: vi.fn(async () => ({ ok: true })), komodoDeploy: vi.fn(async () => ({ ok: true })), komodoDestroyDeployment: vi.fn(async () => ({ ok: true })), komodoDeleteDeployment: vi.fn(async () => ({ ok: true })), komodoInspectHealthy: vi.fn(async () => ({ ok: true, state: 'running' })), komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })), komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true })) })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(async () => 0), rconStatusResult: vi.fn(async (server: string) => ({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: '', status: { players: 0, map: server.startsWith('t6_') ? 'zm_transit' : server.includes('_moon_') ? 'zombie_moon' : 'zombie_theater', gametype: null, roster: [] } })) })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); let fixture: Fixture; const entries = [ { game: 't5', mapSlug: 'theater', count: 2 }, { game: 't5', mapSlug: 'moon', count: 1 }, { game: 't6', mapSlug: 'town', count: 2 } ]; beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); vi.stubEnv('KOMODO_API_KEY', 'private-api-key'); vi.stubEnv('KOMODO_API_SECRET', 'private-api-secret'); vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host'); vi.stubEnv('RCON_PASSWORD', 'private-rcon-password'); vi.stubEnv('SERVER_PORT_MIN', '28960'); vi.stubEnv('SERVER_PORT_MAX', '28980'); vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1'); }); afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); async function previewLineup() { const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { previewCollection } = await import('../../src/lib/server/fleet/collections.js'); const blueprint = saveBlueprint({ name: 'Friday Zombies', entries }); return { blueprint, preview: previewCollection(blueprint.id) }; } function keysFor(preview: CollectionPreview) { return Object.fromEntries(preview.servers.filter(server => server.name).map(server => [server.name!, 'key-' + server.name])); } describe('multi-server blueprints', () => { it('shows the saved startup limit in new previews and existing collection progress', async () => { const { blueprint, preview } = await previewLineup(); const { queueCollection, getCollection, previewCollection } = await import('../../src/lib/server/fleet/collections.js'); const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 }); expect(previewCollection(blueprint.id).launchConcurrency).toBe(3); expect(getCollection(result.collectionId!)?.queue.concurrency).toBe(3); }); it('saves a mixed-game lineup and previews unique slots and global ports without reserving them', async () => { const { blueprint, preview } = await previewLineup(); const { db } = await import('../../src/lib/server/db/index.js'); const { previewCreation } = await import('../../src/lib/server/fleet/create.js'); expect(blueprint.totalServers).toBe(5); expect(blueprint.entries.map(entry => entry.count)).toEqual([2, 1, 2]); expect(blueprint).not.toHaveProperty('recipe'); const stored = db().prepare('SELECT recipe FROM server_blueprints WHERE id = ?').get(blueprint.id) as { recipe: string }; expect(JSON.parse(stored.recipe)).toEqual({ format: 3, entries }); expect(preview.ready).toBe(true); expect(preview.launchConcurrency).toBe(1); expect(preview.servers.map(server => server.name)).toEqual(['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1', 't6_zombie_town_1', 't6_zombie_town_2']); expect(preview.servers.map(server => server.port)).toEqual([28960, 28961, 28962, 28963, 28964]); expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]); expect(db().prepare('SELECT * FROM server_keys').all()).toEqual([]); expect(JSON.stringify(preview)).not.toContain('private-'); expect(preview.entries[0].recipe.files[0]).not.toHaveProperty('content'); expect(() => previewCreation({ blueprintId: blueprint.id })).toThrow(/multiple servers/); }); it('reads a legacy one-server blueprint as a map selection and saves edits without snapshots', async () => { const { captureRecipe, listBlueprints, updateBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); const { previewCreation } = await import('../../src/lib/server/fleet/create.js'); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const artifact = createScript({ name: 'Telemetry', targetRelPath: 'telemetry.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('legacy telemetry') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(artifact.version.id, artifact.script.id); const recipe = captureRecipe('t5', 'theater'); db().prepare('INSERT INTO server_blueprints (id,name,created_at,recipe) VALUES (?, ?, ?, ?)').run('legacy', 'Old favorite', new Date().toISOString(), JSON.stringify(recipe)); const current = uploadNewVersion(artifact.script.id, Buffer.from('current telemetry')); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(current.id, artifact.script.id); closeDb(); const old = listBlueprints()[0]; expect(old.totalServers).toBe(1); expect(old.entries).toEqual([{ count: 1, game: 't5', mapSlug: 'theater', mapName: 'Kino Der Toten' }]); expect(old).not.toHaveProperty('recipe'); expect(previewCreation({ blueprintId: 'legacy' }).recipe.files).toContainEqual(expect.objectContaining({ path: 'telemetry.gsc', versionId: current.id })); const edited = updateBlueprint('legacy', { name: 'Whole lineup', entries }); expect(edited.totalServers).toBe(5); expect(edited.entries[0]).toEqual({ ...old.entries[0], count: 2 }); const stored = db().prepare('SELECT recipe FROM server_blueprints WHERE id = ?').get('legacy') as { recipe: string }; expect(JSON.parse(stored.recipe)).toEqual({ format: 3, entries }); expect(listBlueprints()).toHaveLength(1); }); it('uses current promotions for legacy collection blueprints while retaining counts and exact T6 variants', async () => { const { captureRecipe, listBlueprints } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const { processCreations } = await import('../../src/lib/server/fleet/create.js'); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const artifact = createScript({ name: 'Legacy telemetry', targetRelPath: 'telemetry.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('legacy telemetry') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(artifact.version.id, artifact.script.id); const legacyEntries = entries.map(({ count, game, mapSlug }) => ({ count, recipe: captureRecipe(game as 't5' | 't6', mapSlug) })); db().prepare('INSERT INTO server_blueprints (id,name,created_at,recipe) VALUES (?, ?, ?, ?)').run('legacy-lineup', 'Legacy lineup', new Date().toISOString(), JSON.stringify({ format: 2, entries: legacyEntries })); const current = uploadNewVersion(artifact.script.id, Buffer.from('current telemetry')); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(current.id, artifact.script.id); closeDb(); expect(listBlueprints()[0].entries.map(({ mapName: _mapName, ...entry }) => entry)).toEqual(entries); const preview = previewCollection('legacy-lineup'); expect(preview.entries[0].recipe.files).toContainEqual(expect.objectContaining({ path: 'telemetry.gsc', versionId: current.id })); expect(preview.entries[2].recipe.mapSlug).toBe('town'); const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); for (let pass = 0; pass <= preview.servers.length; pass++) await processCreations(); expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, done: 5 }); for (const name of ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1']) { expect(await readFile(join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/telemetry.gsc'), 'utf8')).toBe('current telemetry'); } expect(await readFile(join(fixture.t6, 't6_zombie_town_2', 'configs/electric_cherry_map.cfg'), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit'); }); it.each([[], [{ game: 't5', mapSlug: 'theater', count: 0 }], [{ game: 't5', mapSlug: 'theater', count: 21 }], [{ game: 't5', mapSlug: 'theater', count: 1.5 }], [{ game: 't5', mapSlug: 'theater', count: '2' }], [{ game: 't5', mapSlug: 'town', count: 1 }], [entries[0], entries[0]]].map(input => ({ input })))('rejects an invalid lineup: $input', async ({ input }) => { const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); expect(() => saveBlueprint({ name: 'Invalid', entries: input })).toThrow(); }); it('reports insufficient capacity without preventing blueprint creation', async () => { vi.stubEnv('SERVER_PORT_MAX', '28961'); const { blueprint, preview } = await previewLineup(); const { queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const { db } = await import('../../src/lib/server/db/index.js'); expect(blueprint.totalServers).toBe(5); expect(preview.ready).toBe(false); expect(preview.servers.filter(server => server.port === null)).toHaveLength(3); expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/not enough free game ports/); expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]); }); }); describe('atomic collection launches', () => { it('fills the target shortfall and counts registered/in-flight identities only once', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); for (let i = 0; i < 2; i++) expect((await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing-key' })).ok).toBe(true); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const blueprint = saveBlueprint({ name: 'Friday targets', entries: [ { game: 't5', mapSlug: 'theater', count: 4 }, { game: 't5', mapSlug: 'factory', count: 2 } ] }); const preview = previewCollection(blueprint.id); expect(preview.entries.map(({ count, existing, missing }) => ({ count, existing, missing }))).toEqual([ { count: 4, existing: 2, missing: 2 }, { count: 2, existing: 0, missing: 2 } ]); expect(preview.servers.map(server => server.name)).toEqual(['t5_zombie_theater_3', 't5_zombie_theater_4', 't5_zombie_factory_1', 't5_zombie_factory_2']); const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); expect(getCollection(queued.collectionId!)?.collection.total).toBe(4); expect(previewCollection(blueprint.id).servers).toEqual([]); }); it('keeps excess and unrelated servers, including stopped servers, without requiring launch settings for a no-op', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); for (let i = 0; i < 2; i++) await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing-key' }); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare("UPDATE servers SET container_state = 'stopped'").run(); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const { config } = await import('../../src/lib/server/config.js'); // Fixture-only configuration changes simulate missing integrations after creation. Object.assign(config.rcon, { password: '' }); Object.assign(config.komodo, { targetServer: '' }); const blueprint = saveBlueprint({ name: 'One Kino', entries: [{ game: 't5', mapSlug: 'theater', count: 1 }] }); const preview = previewCollection(blueprint.id); expect(preview.entries[0]).toMatchObject({ existing: 2, missing: 0 }); expect(preview).toMatchObject({ ready: true, blockers: [], servers: [] }); expect(queueCollection({ previewId: preview.id })).toEqual({ ok: true, collectionId: null }); expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]); expect(db().prepare('SELECT * FROM servers').all()).toHaveLength(2); }); it('rechecks shrinking deficits at submission and rejects an older additive preview', async () => { const { preview, blueprint } = await previewLineup(); const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); const { queueCollection, previewCollection } = await import('../../src/lib/server/fleet/collections.js'); const { db } = await import('../../src/lib/server/db/index.js'); queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing-key' }); expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/changed/); expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1); const fresh = previewCollection(blueprint.id); expect(fresh.servers).toHaveLength(4); const extra = [...fresh.servers, { ...fresh.servers[0], name: 't5_zombie_theater_20', port: 28980 }]; db().prepare('UPDATE server_collection_previews SET servers = ? WHERE id = ?').run(JSON.stringify(extra), fresh.id); expect(() => queueCollection({ previewId: fresh.id, keys: keysFor(fresh) })).toThrow(/changed/); }); it('matches exact T6 variants and excludes failed creations and pending destructions', async () => { const { createServer, queueCreateServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); const { finishDeployment } = await import('../../src/lib/server/fleet/deployments.js'); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { previewCollection } = await import('../../src/lib/server/fleet/collections.js'); const farm = await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'farm', plutoKey: 'farm-key' }); const town = await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'town-key' }); if (!farm.ok || !town.ok) throw new Error('Fixture creation failed'); destroyServer(town.name); const failed = queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'failed-key' }); if (!failed.ok) throw new Error(failed.message); finishDeployment(failed.deploymentId, { ok: false, message: 'Fixture failure' }); const blueprint = saveBlueprint({ name: 'Town target', entries: [{ game: 't6', mapSlug: 'town', count: 1 }] }); const preview = previewCollection(blueprint.id); expect(preview.entries[0]).toMatchObject({ existing: 0, missing: 1 }); expect(preview.servers[0].name).toBe('t6_zombie_town_2'); }); it('queues all five servers, persists one group, and completes each independently', async () => { const { preview } = await previewLineup(); const { queueCollection, getCollection, listCollections } = await import('../../src/lib/server/fleet/collections.js'); const { processCreations } = await import('../../src/lib/server/fleet/create.js'); const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); const { closeDb } = await import('../../src/lib/server/db/index.js'); const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); expect(komodoCreateDeployment).not.toHaveBeenCalled(); expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, active: 5, done: 0, failed: 0, queued: 5, provisioning: 0, verifying: 0 }); expect(getCollection(queued.collectionId!)?.servers.map(server => server.queuePosition)).toEqual([1, 2, 3, 4, 5]); closeDb(); await processCreations(); expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ queued: 4, provisioning: 0, verifying: 1, done: 0 }); expect(getCollection(queued.collectionId!)?.queue).toMatchObject({ concurrency: 1, queued: 4, active: [{ server: preview.servers[0].name, state: 'awaiting-health' }] }); expect(getCollection(queued.collectionId!)?.servers.map(server => server.queuePosition)).toEqual([null, 1, 2, 3, 4]); for (let pass = 0; pass < preview.servers.length; pass++) await processCreations(); const detail = getCollection(queued.collectionId!)!; expect(detail.collection).toMatchObject({ total: 5, active: 0, done: 5, failed: 0 }); expect(listCollections()).toHaveLength(1); expect(komodoCreateDeployment).toHaveBeenCalledTimes(5); expect(getDeployment(detail.servers[0].id)?.collection).toEqual({ id: queued.collectionId, name: 'Friday Zombies' }); expect(await readFile(join(fixture.t6, 't6_zombie_town_2', 'configs/electric_cherry_map.cfg'), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit'); expect(JSON.stringify(detail)).not.toContain('key-t5'); }); it('rolls back all earlier jobs and key changes if the final server has no valid key', async () => { const { preview } = await previewLineup(); const { queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { saveServerKey, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); saveServerKey(preview.servers[0].name!, 'original-key'); const keys = keysFor(preview); delete keys[preview.servers[4].name!]; expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/No saved key/); for (const table of ['server_launch_collections', 'server_collection_members', 'server_creation_jobs', 'server_deployments', 'server_launch_previews', 'server_launch_recipes']) { expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]); } expect(getServerKey(preview.servers[0].name!)).toBe('original-key'); expect(db().prepare('SELECT * FROM server_keys').all()).toHaveLength(1); keys[preview.servers[4].name!] = 'invalid key'; expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/without whitespace/); expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]); expect(queueCollection({ previewId: preview.id, keys: keysFor(preview) }).ok).toBe(true); }); it('rejects changed capacity before saving keys or partially queueing the lineup', async () => { const { preview } = await previewLineup(); const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); const { queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const { db } = await import('../../src/lib/server/db/index.js'); queueCreateServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', plutoKey: 'other-server-key' }); expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/availability changed/); expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1); expect(db().prepare('SELECT * FROM server_keys').all()).toHaveLength(1); expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]); }); it('reuses saved keys and returns the same collection on resubmission after restart and expiry', async () => { const { preview, blueprint } = await previewLineup(); const { queueCollection, previewCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); for (const server of preview.servers) saveServerKey(server.name!, `saved-${server.name}`); const ready = previewCollection(blueprint.id); expect(ready.servers.every(server => server.hasSavedKey)).toBe(true); const queued = queueCollection({ previewId: ready.id }); db().prepare("UPDATE server_collection_previews SET expires_at = '2000-01-01T00:00:00Z' WHERE id = ?").run(ready.id); closeDb(); expect(queueCollection({ previewId: ready.id })).toEqual(queued); expect(getCollection(queued.collectionId!)?.servers).toHaveLength(5); expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(5); const next = previewCollection(blueprint.id); expect(next.servers).toEqual([]); expect(next.entries.map(entry => entry.existing)).toEqual([2, 1, 2]); expect(queueCollection({ previewId: next.id })).toEqual({ ok: true, collectionId: null }); expect(db().prepare('SELECT * FROM server_launch_collections').all()).toHaveLength(1); }); it('keeps the reviewed lineup after blueprint edits and deletion while using current promotions', async () => { const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { script, version } = createScript({ name: 'Classic perks', kind: 'script', scopeType: 'game', scopeValue: 't5', targetRelPath: 'perks.gsc', content: Buffer.from('original') }); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id); const { preview, blueprint } = await previewLineup(); const { updateBlueprint, deleteBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const { processCreations } = await import('../../src/lib/server/fleet/create.js'); const newer = uploadNewVersion(script.id, Buffer.from('new version')); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newer.id, script.id); updateBlueprint(blueprint.id, { name: 'Changed', entries: [{ ...entries[0], count: 1 }] }); deleteBlueprint(blueprint.id); const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); const queuedDetail = getCollection(queued.collectionId!)!; const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js'); for (const server of queuedDetail.servers.filter(server => server.server.startsWith('t5_'))) { expect(recipeForDeployment(server.id)?.recipe.files).toContainEqual(expect.objectContaining({ path: 'perks.gsc', versionId: newer.id })); } const newest = uploadNewVersion(script.id, Buffer.from('promoted while queued')); db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newest.id, script.id); uploadNewVersion(script.id, Buffer.from('unpromoted draft')); for (let pass = 0; pass <= preview.servers.length; pass++) await processCreations(); expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ name: 'Friday Zombies', total: 5, done: 5 }); for (const name of ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1']) { expect(await readFile(join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('promoted while queued'); } }); it('keeps a provisioning failure separate while the other servers complete', async () => { const { preview } = await previewLineup(); const { queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const { processCreations } = await import('../../src/lib/server/fleet/create.js'); const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoCreateDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'Host rejected this container' }); const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); await processCreations(); expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ queued: 4, failed: 1, done: 0 }); expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); for (let pass = 0; pass < preview.servers.length; pass++) await processCreations(); const detail = getCollection(queued.collectionId!)!; expect(detail.collection).toMatchObject({ total: 5, active: 0, done: 4, failed: 1 }); expect(detail.servers[0]).toMatchObject({ state: 'failed', error: expect.stringContaining('Host rejected') }); expect(detail.servers.slice(1).every(server => server.state === 'done')).toBe(true); }); it('validates collection API payloads and rejects an unused expired preview', async () => { const { preview, blueprint } = await previewLineup(); const { POST: singlePreview } = await import('../../src/routes/api/deployments/preview/+server.js'); const oldEndpoint = await singlePreview({ request: new Request('http://localhost/api/deployments/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ blueprintId: blueprint.id }) }) }); expect(oldEndpoint.status).toBe(400); expect((await oldEndpoint.json()).error).toContain('target counts'); const { db } = await import('../../src/lib/server/db/index.js'); const { queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const { POST } = await import('../../src/routes/api/deployment-collections/+server.js'); for (const body of [null, [], {}, { previewId: preview.id, keys: [] }, { previewId: preview.id, keys: { wrong: 'key' } }]) { const response = await POST({ request: new Request('http://localhost/api/deployment-collections', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }) }); expect(response.status).toBe(400); } db().prepare("UPDATE server_collection_previews SET expires_at = '2000-01-01T00:00:00Z' WHERE id = ?").run(preview.id); expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/expired/); expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]); }); }); describe('matching a blueprint exactly', () => { async function blueprint(entries: { game: string; mapSlug: string; count: number }[]) { const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); return saveBlueprint({ name: 'Exact lineup', entries }); } it('keeps the lowest deployed lobbies, removes excess and unrelated maps, and adds only the shortfall', async () => { const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js'); for (const mapSlug of ['theater', 'theater', 'moon']) await createServer({ gameMode: 'zombie', mapSlug, plutoKey: 'existing' }); const { db } = await import('../../src/lib/server/db/index.js'); const before = db().prepare("SELECT * FROM servers WHERE name = 't5_zombie_theater_1'").get(); const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }, { game: 't6', mapSlug: 'town', count: 1 }]); const preview = previewCollection(plan.id, 'exact'); expect(preview.retainedServers).toEqual(['t5_zombie_theater_1']); expect(preview.removals.map(server => server.name)).toEqual(['t5_zombie_moon_1', 't5_zombie_theater_2']); expect(preview.servers.map(server => server.name)).toEqual(['t6_zombie_town_1']); const { komodoDestroyDeployment, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoCreateDeployment).mockClear(); const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); expect(komodoDestroyDeployment).not.toHaveBeenCalled(); expect(komodoCreateDeployment).not.toHaveBeenCalled(); expect(getCollection(result.collectionId!)?.collection).toMatchObject({ mode: 'exact', total: 3, active: 3, queued: 3 }); vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }).mockResolvedValueOnce({ ok: false, reason: 'gone' }); await processDestructions(); await processCreations(); await processCreations(); expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: 3, done: 3, active: 0 }); expect(getCollection(result.collectionId!)?.servers.filter(server => server.action === 'destroy')).toHaveLength(2); expect(db().prepare("SELECT * FROM servers WHERE name = 't5_zombie_theater_1'").get()).toEqual(before); expect(db().prepare('SELECT name FROM servers ORDER BY name').all()).toEqual([{ name: 't5_zombie_theater_1' }, { name: 't6_zombie_town_1' }]); expect(komodoDestroyDeployment).not.toHaveBeenCalledWith('t5_zombie_theater_1'); }); it('supports removal-only plans, shares existing removals, and records retained servers', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); for (let i = 0; i < 2; i++) await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' }); const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const first = previewCollection(plan.id, 'exact'); expect(first.servers).toEqual([]); const a = queueCollection({ previewId: first.id }); const b = queueCollection({ previewId: previewCollection(plan.id, 'exact').id }); const firstDetail = getCollection(a.collectionId!)!; expect(firstDetail.retainedServers).toEqual(['t5_zombie_theater_1']); expect(firstDetail.servers).toHaveLength(1); expect(getCollection(b.collectionId!)?.servers[0].id).toBe(firstDetail.servers[0].id); expect(firstDetail.servers[0]).toMatchObject({ action: 'destroy', state: 'waiting', queuePosition: null }); const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); expect(getDeployment(firstDetail.servers[0].id)?.collection).toBeTruthy(); }); it('cancels unstarted excess creations without booting them and retains their saved keys', async () => { const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); const first = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'first' }); const second = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'second' }); if (!first.ok || !second.ok) throw new Error('Fixture queue failed'); const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const queued = queueCollection({ previewId: previewCollection(plan.id, 'exact').id }); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(second.deploymentId)).toEqual({ state: 'cancelled' }); expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(first.deploymentId)).toEqual({ state: 'queued' }); expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ done: 1, total: 1, active: 0 }); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); expect(getServerKey(second.name)).toBe('second'); const { komodoCreateDeployment, komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); expect(komodoCreateDeployment).not.toHaveBeenCalled(); expect(komodoDestroyDeployment).not.toHaveBeenCalled(); }); it('rejects fleet changes even when the create shortfall stays the same', async () => { const { createServer, queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' }); const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const preview = previewCollection(plan.id, 'exact'); queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'new' }); expect(() => queueCollection({ previewId: preview.id })).toThrow(/fleet changed/); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]); }); it('rolls back removals and creation cancellations when a required key is missing', async () => { const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); const old = queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' }); if (!old.ok) throw new Error(old.message); const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const preview = previewCollection(plan.id, 'exact'); expect(() => queueCollection({ previewId: preview.id })).toThrow(/No saved key/); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.deploymentId)).toEqual({ state: 'queued' }); for (const table of ['server_collection_removals', 'server_launch_collections', 'server_creation_dependencies']) expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]); expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]); }); it('reuses a full fleet port only after empty-server removal, across restart and repeated submission', async () => { vi.stubEnv('SERVER_PORT_MAX', '28960'); const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js'); await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' }); await processCreations(); const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const preview = previewCollection(plan.id, 'exact'); expect(preview).toMatchObject({ ready: true, servers: [{ port: 28960 }] }); const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); expect(queueCollection({ previewId: preview.id })).toEqual(result); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); const { komodoCreateDeployment, komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoCreateDeployment).mockClear(); for (const players of [2, null]) { vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(players); await processDestructions(); await processCreations(); expect(komodoDestroyDeployment).not.toHaveBeenCalled(); expect(komodoCreateDeployment).not.toHaveBeenCalled(); expect(getCollection(result.collectionId!)?.servers.find(server => server.action === 'create')?.stage).toContain('Waiting for removal'); } vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }); await processDestructions(); await processCreations(); await processCreations(); expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: 2, done: 2, active: 0 }); }); it('does not reuse a port after a removal is cancelled', async () => { vi.stubEnv('SERVER_PORT_MAX', '28960'); const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js'); await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' }); await processCreations(); const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const preview = previewCollection(plan.id, 'exact'); const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js'); expect(cancelRemoval(getCollection(result.collectionId!)!.servers.find(server => server.action === 'destroy')!.id)).toBe(true); const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoCreateDeployment).mockClear(); await processCreations(); expect(komodoCreateDeployment).not.toHaveBeenCalled(); expect(getCollection(result.collectionId!)?.collection.failed).toBe(1); expect(getCollection(result.collectionId!)?.servers.find(server => server.action === 'create')?.stage).toContain('(cancelled)'); }); it('allows independent launches to pass a failed removal without exceeding the startup limit', async () => { const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js'); await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' }); await processCreations(); const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 2 }]); const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const preview = previewCollection(plan.id, 'exact'); const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) }); const { komodoCreateDeployment, komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoCreateDeployment).mockClear(); vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'Could not stop container' }); await processDestructions(); await processCreations(); const creates = getCollection(result.collectionId!)!.servers.filter(server => server.action === 'create'); expect(creates[0]).toMatchObject({ state: 'queued', stage: expect.stringContaining('(failed)') }); expect(creates[1]).toMatchObject({ state: 'awaiting-health' }); expect(komodoCreateDeployment).toHaveBeenCalledTimes(1); expect(getCollection(result.collectionId!)!.collection).toMatchObject({ failed: 1, queued: 1, verifying: 1 }); }); it('rejects insufficient final capacity before cancelling any existing creation', async () => { vi.stubEnv('SERVER_PORT_MAX', '28960'); const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js'); const old = queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' }); if (!old.ok) throw new Error(old.message); const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 2 }]); const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const preview = previewCollection(plan.id, 'exact'); expect(preview.ready).toBe(false); expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/not enough free game ports/); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.deploymentId)).toEqual({ state: 'queued' }); expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]); }); it('requires a new review when a pending removal is cancelled', async () => { const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js'); const old = await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' }); if (!old.ok) throw new Error(old.message); const removal = destroyServer(old.name); if (!removal.ok) throw new Error(removal.message); const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]); const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const preview = previewCollection(plan.id, 'exact'); const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js'); cancelRemoval(removal.deploymentId!); expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/fleet changed/); expect(previewCollection(plan.id, 'exact')).toMatchObject({ servers: [], removals: [], retainedServers: [old.name] }); }); it('keeps T6 variants distinct and leaves a matching fleet untouched', async () => { const { createServer } = await import('../../src/lib/server/fleet/create.js'); await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'town' }); const plan = await blueprint([{ game: 't6', mapSlug: 'town', count: 1 }]); const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const noOp = previewCollection(plan.id, 'exact'); expect(noOp).toMatchObject({ retainedServers: ['t6_zombie_town_1'], removals: [], servers: [] }); expect(queueCollection({ previewId: noOp.id })).toEqual({ ok: true, collectionId: null }); await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'farm', plutoKey: 'farm' }); const preview = previewCollection(plan.id, 'exact'); expect(preview.retainedServers).toEqual(['t6_zombie_town_1']); expect(preview.removals.map(server => server.name)).toEqual(['t6_zombie_farm_1']); }); it('defaults old previews and collections to fill and validates the preview mode API', async () => { const { blueprint: plan } = await previewLineup(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); db().exec('ALTER TABLE server_collection_previews DROP COLUMN mode'); db().exec('ALTER TABLE server_collection_previews DROP COLUMN fleet_snapshot'); db().exec('ALTER TABLE server_launch_collections DROP COLUMN mode'); // Recreate the older schema, which predates the history preservation trigger. db().exec('DROP TRIGGER preserve_deleted_server_history'); db().exec('ALTER TABLE server_launch_collections DROP COLUMN retained_servers'); closeDb(); const { previewCollection } = await import('../../src/lib/server/fleet/collections.js'); expect(previewCollection(plan.id).mode).toBe('fill'); const { POST } = await import('../../src/routes/api/deployment-collections/preview/+server.js'); for (const mode of ['exact', 'fill', 'invalid', null, true]) { const response = await POST({ request: new Request('http://localhost/api/deployment-collections/preview', { method: 'POST', body: JSON.stringify({ blueprintId: plan.id, mode }) }) }); expect(response.status).toBe(mode === 'exact' || mode === 'fill' ? 200 : 400); if (response.ok) expect((await response.json()).mode).toBe(mode); } }); });