Files
electric-cherry/tests/unit/deployment-collections.test.ts
T
josh 981fb27c1c Preserve deleted server history
Retain historical server identities, results, and evidence after deletion. Mark archived entries as Deleted and prevent old work from targeting servers that reuse the same name.

Fixes #16.
2026-09-27 21:48:55 -04:00

605 lines
46 KiB
TypeScript

import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { readFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import type { CollectionPreview } from '../../src/lib/deployment-plan.js';
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
komodoCreateDeployment: vi.fn(async () => ({ ok: true })),
komodoDeploy: vi.fn(async () => ({ ok: true })),
komodoDestroyDeployment: vi.fn(async () => ({ ok: true })),
komodoDeleteDeployment: vi.fn(async () => ({ ok: true })),
komodoInspectHealthy: vi.fn(async () => ({ ok: true, state: 'running' })),
komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })),
komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true }))
}));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
rconGetPlayerCount: vi.fn(async () => 0),
rconStatusResult: vi.fn(async (server: string) => ({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: '',
status: { players: 0, map: server.startsWith('t6_') ? 'zm_transit' : server.includes('_moon_') ? 'zombie_moon' : 'zombie_theater', gametype: null, roster: [] } }))
}));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
let fixture: Fixture;
const entries = [
{ game: 't5', mapSlug: 'theater', count: 2 },
{ game: 't5', mapSlug: 'moon', count: 1 },
{ game: 't6', mapSlug: 'town', count: 2 }
];
beforeEach(async () => {
vi.resetModules(); vi.clearAllMocks();
fixture = await newFixture([]);
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
vi.stubEnv('KOMODO_API_KEY', 'private-api-key');
vi.stubEnv('KOMODO_API_SECRET', 'private-api-secret');
vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host');
vi.stubEnv('RCON_PASSWORD', 'private-rcon-password');
vi.stubEnv('SERVER_PORT_MIN', '28960');
vi.stubEnv('SERVER_PORT_MAX', '28980');
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1');
});
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
async function previewLineup() {
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection } = await import('../../src/lib/server/fleet/collections.js');
const blueprint = saveBlueprint({ name: 'Friday Zombies', entries });
return { blueprint, preview: previewCollection(blueprint.id) };
}
function keysFor(preview: CollectionPreview) {
return Object.fromEntries(preview.servers.filter(server => server.name).map(server => [server.name!, 'key-' + server.name]));
}
describe('multi-server blueprints', () => {
it('shows the saved startup limit in new previews and existing collection progress', async () => {
const { blueprint, preview } = await previewLineup();
const { queueCollection, getCollection, previewCollection } = await import('../../src/lib/server/fleet/collections.js');
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
expect(previewCollection(blueprint.id).launchConcurrency).toBe(3);
expect(getCollection(result.collectionId!)?.queue.concurrency).toBe(3);
});
it('saves a mixed-game lineup and previews unique slots and global ports without reserving them', async () => {
const { blueprint, preview } = await previewLineup();
const { db } = await import('../../src/lib/server/db/index.js');
const { previewCreation } = await import('../../src/lib/server/fleet/create.js');
expect(blueprint.totalServers).toBe(5);
expect(blueprint.entries.map(entry => entry.count)).toEqual([2, 1, 2]);
expect(blueprint).not.toHaveProperty('recipe');
const stored = db().prepare('SELECT recipe FROM server_blueprints WHERE id = ?').get(blueprint.id) as { recipe: string };
expect(JSON.parse(stored.recipe)).toEqual({ format: 3, entries });
expect(preview.ready).toBe(true);
expect(preview.launchConcurrency).toBe(1);
expect(preview.servers.map(server => server.name)).toEqual(['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1', 't6_zombie_town_1', 't6_zombie_town_2']);
expect(preview.servers.map(server => server.port)).toEqual([28960, 28961, 28962, 28963, 28964]);
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]);
expect(db().prepare('SELECT * FROM server_keys').all()).toEqual([]);
expect(JSON.stringify(preview)).not.toContain('private-');
expect(preview.entries[0].recipe.files[0]).not.toHaveProperty('content');
expect(() => previewCreation({ blueprintId: blueprint.id })).toThrow(/multiple servers/);
});
it('reads a legacy one-server blueprint as a map selection and saves edits without snapshots', async () => {
const { captureRecipe, listBlueprints, updateBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { previewCreation } = await import('../../src/lib/server/fleet/create.js');
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const artifact = createScript({ name: 'Telemetry', targetRelPath: 'telemetry.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('legacy telemetry') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(artifact.version.id, artifact.script.id);
const recipe = captureRecipe('t5', 'theater');
db().prepare('INSERT INTO server_blueprints (id,name,created_at,recipe) VALUES (?, ?, ?, ?)').run('legacy', 'Old favorite', new Date().toISOString(), JSON.stringify(recipe));
const current = uploadNewVersion(artifact.script.id, Buffer.from('current telemetry'));
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(current.id, artifact.script.id);
closeDb();
const old = listBlueprints()[0];
expect(old.totalServers).toBe(1);
expect(old.entries).toEqual([{ count: 1, game: 't5', mapSlug: 'theater', mapName: 'Kino Der Toten' }]);
expect(old).not.toHaveProperty('recipe');
expect(previewCreation({ blueprintId: 'legacy' }).recipe.files).toContainEqual(expect.objectContaining({ path: 'telemetry.gsc', versionId: current.id }));
const edited = updateBlueprint('legacy', { name: 'Whole lineup', entries });
expect(edited.totalServers).toBe(5);
expect(edited.entries[0]).toEqual({ ...old.entries[0], count: 2 });
const stored = db().prepare('SELECT recipe FROM server_blueprints WHERE id = ?').get('legacy') as { recipe: string };
expect(JSON.parse(stored.recipe)).toEqual({ format: 3, entries });
expect(listBlueprints()).toHaveLength(1);
});
it('uses current promotions for legacy collection blueprints while retaining counts and exact T6 variants', async () => {
const { captureRecipe, listBlueprints } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const artifact = createScript({ name: 'Legacy telemetry', targetRelPath: 'telemetry.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('legacy telemetry') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(artifact.version.id, artifact.script.id);
const legacyEntries = entries.map(({ count, game, mapSlug }) => ({ count, recipe: captureRecipe(game as 't5' | 't6', mapSlug) }));
db().prepare('INSERT INTO server_blueprints (id,name,created_at,recipe) VALUES (?, ?, ?, ?)').run('legacy-lineup', 'Legacy lineup', new Date().toISOString(), JSON.stringify({ format: 2, entries: legacyEntries }));
const current = uploadNewVersion(artifact.script.id, Buffer.from('current telemetry'));
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(current.id, artifact.script.id);
closeDb();
expect(listBlueprints()[0].entries.map(({ mapName: _mapName, ...entry }) => entry)).toEqual(entries);
const preview = previewCollection('legacy-lineup');
expect(preview.entries[0].recipe.files).toContainEqual(expect.objectContaining({ path: 'telemetry.gsc', versionId: current.id }));
expect(preview.entries[2].recipe.mapSlug).toBe('town');
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
for (let pass = 0; pass <= preview.servers.length; pass++) await processCreations();
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, done: 5 });
for (const name of ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1']) {
expect(await readFile(join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/telemetry.gsc'), 'utf8')).toBe('current telemetry');
}
expect(await readFile(join(fixture.t6, 't6_zombie_town_2', 'configs/electric_cherry_map.cfg'), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit');
});
it.each([[], [{ game: 't5', mapSlug: 'theater', count: 0 }], [{ game: 't5', mapSlug: 'theater', count: 21 }],
[{ game: 't5', mapSlug: 'theater', count: 1.5 }], [{ game: 't5', mapSlug: 'theater', count: '2' }],
[{ game: 't5', mapSlug: 'town', count: 1 }], [entries[0], entries[0]]].map(input => ({ input })))('rejects an invalid lineup: $input', async ({ input }) => {
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
expect(() => saveBlueprint({ name: 'Invalid', entries: input })).toThrow();
});
it('reports insufficient capacity without preventing blueprint creation', async () => {
vi.stubEnv('SERVER_PORT_MAX', '28961');
const { blueprint, preview } = await previewLineup();
const { queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const { db } = await import('../../src/lib/server/db/index.js');
expect(blueprint.totalServers).toBe(5);
expect(preview.ready).toBe(false);
expect(preview.servers.filter(server => server.port === null)).toHaveLength(3);
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/not enough free game ports/);
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
});
});
describe('atomic collection launches', () => {
it('fills the target shortfall and counts registered/in-flight identities only once', async () => {
const { createServer } = await import('../../src/lib/server/fleet/create.js');
for (let i = 0; i < 2; i++) expect((await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing-key' })).ok).toBe(true);
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const blueprint = saveBlueprint({ name: 'Friday targets', entries: [
{ game: 't5', mapSlug: 'theater', count: 4 }, { game: 't5', mapSlug: 'factory', count: 2 }
] });
const preview = previewCollection(blueprint.id);
expect(preview.entries.map(({ count, existing, missing }) => ({ count, existing, missing }))).toEqual([
{ count: 4, existing: 2, missing: 2 }, { count: 2, existing: 0, missing: 2 }
]);
expect(preview.servers.map(server => server.name)).toEqual(['t5_zombie_theater_3', 't5_zombie_theater_4', 't5_zombie_factory_1', 't5_zombie_factory_2']);
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
expect(getCollection(queued.collectionId!)?.collection.total).toBe(4);
expect(previewCollection(blueprint.id).servers).toEqual([]);
});
it('keeps excess and unrelated servers, including stopped servers, without requiring launch settings for a no-op', async () => {
const { createServer } = await import('../../src/lib/server/fleet/create.js');
for (let i = 0; i < 2; i++) await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing-key' });
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE servers SET container_state = 'stopped'").run();
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const { config } = await import('../../src/lib/server/config.js');
// Fixture-only configuration changes simulate missing integrations after creation.
Object.assign(config.rcon, { password: '' });
Object.assign(config.komodo, { targetServer: '' });
const blueprint = saveBlueprint({ name: 'One Kino', entries: [{ game: 't5', mapSlug: 'theater', count: 1 }] });
const preview = previewCollection(blueprint.id);
expect(preview.entries[0]).toMatchObject({ existing: 2, missing: 0 });
expect(preview).toMatchObject({ ready: true, blockers: [], servers: [] });
expect(queueCollection({ previewId: preview.id })).toEqual({ ok: true, collectionId: null });
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
expect(db().prepare('SELECT * FROM servers').all()).toHaveLength(2);
});
it('rechecks shrinking deficits at submission and rejects an older additive preview', async () => {
const { preview, blueprint } = await previewLineup();
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
const { queueCollection, previewCollection } = await import('../../src/lib/server/fleet/collections.js');
const { db } = await import('../../src/lib/server/db/index.js');
queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing-key' });
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/changed/);
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1);
const fresh = previewCollection(blueprint.id);
expect(fresh.servers).toHaveLength(4);
const extra = [...fresh.servers, { ...fresh.servers[0], name: 't5_zombie_theater_20', port: 28980 }];
db().prepare('UPDATE server_collection_previews SET servers = ? WHERE id = ?').run(JSON.stringify(extra), fresh.id);
expect(() => queueCollection({ previewId: fresh.id, keys: keysFor(fresh) })).toThrow(/changed/);
});
it('matches exact T6 variants and excludes failed creations and pending destructions', async () => {
const { createServer, queueCreateServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
const { finishDeployment } = await import('../../src/lib/server/fleet/deployments.js');
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection } = await import('../../src/lib/server/fleet/collections.js');
const farm = await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'farm', plutoKey: 'farm-key' });
const town = await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'town-key' });
if (!farm.ok || !town.ok) throw new Error('Fixture creation failed');
destroyServer(town.name);
const failed = queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'failed-key' });
if (!failed.ok) throw new Error(failed.message);
finishDeployment(failed.deploymentId, { ok: false, message: 'Fixture failure' });
const blueprint = saveBlueprint({ name: 'Town target', entries: [{ game: 't6', mapSlug: 'town', count: 1 }] });
const preview = previewCollection(blueprint.id);
expect(preview.entries[0]).toMatchObject({ existing: 0, missing: 1 });
expect(preview.servers[0].name).toBe('t6_zombie_town_2');
});
it('queues all five servers, persists one group, and completes each independently', async () => {
const { preview } = await previewLineup();
const { queueCollection, getCollection, listCollections } = await import('../../src/lib/server/fleet/collections.js');
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
const { closeDb } = await import('../../src/lib/server/db/index.js');
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
expect(komodoCreateDeployment).not.toHaveBeenCalled();
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, active: 5, done: 0, failed: 0, queued: 5, provisioning: 0, verifying: 0 });
expect(getCollection(queued.collectionId!)?.servers.map(server => server.queuePosition)).toEqual([1, 2, 3, 4, 5]);
closeDb();
await processCreations();
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ queued: 4, provisioning: 0, verifying: 1, done: 0 });
expect(getCollection(queued.collectionId!)?.queue).toMatchObject({ concurrency: 1, queued: 4, active: [{ server: preview.servers[0].name, state: 'awaiting-health' }] });
expect(getCollection(queued.collectionId!)?.servers.map(server => server.queuePosition)).toEqual([null, 1, 2, 3, 4]);
for (let pass = 0; pass < preview.servers.length; pass++) await processCreations();
const detail = getCollection(queued.collectionId!)!;
expect(detail.collection).toMatchObject({ total: 5, active: 0, done: 5, failed: 0 });
expect(listCollections()).toHaveLength(1);
expect(komodoCreateDeployment).toHaveBeenCalledTimes(5);
expect(getDeployment(detail.servers[0].id)?.collection).toEqual({ id: queued.collectionId, name: 'Friday Zombies' });
expect(await readFile(join(fixture.t6, 't6_zombie_town_2', 'configs/electric_cherry_map.cfg'), 'utf8')).toContain('execgts zm_standard_town.cfg map zm_transit');
expect(JSON.stringify(detail)).not.toContain('key-t5');
});
it('rolls back all earlier jobs and key changes if the final server has no valid key', async () => {
const { preview } = await previewLineup();
const { queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { saveServerKey, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
saveServerKey(preview.servers[0].name!, 'original-key');
const keys = keysFor(preview);
delete keys[preview.servers[4].name!];
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/No saved key/);
for (const table of ['server_launch_collections', 'server_collection_members', 'server_creation_jobs', 'server_deployments', 'server_launch_previews', 'server_launch_recipes']) {
expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
}
expect(getServerKey(preview.servers[0].name!)).toBe('original-key');
expect(db().prepare('SELECT * FROM server_keys').all()).toHaveLength(1);
keys[preview.servers[4].name!] = 'invalid key';
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/without whitespace/);
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]);
expect(queueCollection({ previewId: preview.id, keys: keysFor(preview) }).ok).toBe(true);
});
it('rejects changed capacity before saving keys or partially queueing the lineup', async () => {
const { preview } = await previewLineup();
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
const { queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const { db } = await import('../../src/lib/server/db/index.js');
queueCreateServer({ gameMode: 'zombie', mapSlug: 'cosmodrome', plutoKey: 'other-server-key' });
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/availability changed/);
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1);
expect(db().prepare('SELECT * FROM server_keys').all()).toHaveLength(1);
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
});
it('reuses saved keys and returns the same collection on resubmission after restart and expiry', async () => {
const { preview, blueprint } = await previewLineup();
const { queueCollection, previewCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
for (const server of preview.servers) saveServerKey(server.name!, `saved-${server.name}`);
const ready = previewCollection(blueprint.id);
expect(ready.servers.every(server => server.hasSavedKey)).toBe(true);
const queued = queueCollection({ previewId: ready.id });
db().prepare("UPDATE server_collection_previews SET expires_at = '2000-01-01T00:00:00Z' WHERE id = ?").run(ready.id);
closeDb();
expect(queueCollection({ previewId: ready.id })).toEqual(queued);
expect(getCollection(queued.collectionId!)?.servers).toHaveLength(5);
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(5);
const next = previewCollection(blueprint.id);
expect(next.servers).toEqual([]);
expect(next.entries.map(entry => entry.existing)).toEqual([2, 1, 2]);
expect(queueCollection({ previewId: next.id })).toEqual({ ok: true, collectionId: null });
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toHaveLength(1);
});
it('keeps the reviewed lineup after blueprint edits and deletion while using current promotions', async () => {
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { script, version } = createScript({ name: 'Classic perks', kind: 'script', scopeType: 'game', scopeValue: 't5', targetRelPath: 'perks.gsc', content: Buffer.from('original') });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(version.id, script.id);
const { preview, blueprint } = await previewLineup();
const { updateBlueprint, deleteBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
const newer = uploadNewVersion(script.id, Buffer.from('new version'));
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newer.id, script.id);
updateBlueprint(blueprint.id, { name: 'Changed', entries: [{ ...entries[0], count: 1 }] });
deleteBlueprint(blueprint.id);
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
const queuedDetail = getCollection(queued.collectionId!)!;
const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
for (const server of queuedDetail.servers.filter(server => server.server.startsWith('t5_'))) {
expect(recipeForDeployment(server.id)?.recipe.files).toContainEqual(expect.objectContaining({ path: 'perks.gsc', versionId: newer.id }));
}
const newest = uploadNewVersion(script.id, Buffer.from('promoted while queued'));
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newest.id, script.id);
uploadNewVersion(script.id, Buffer.from('unpromoted draft'));
for (let pass = 0; pass <= preview.servers.length; pass++) await processCreations();
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ name: 'Friday Zombies', total: 5, done: 5 });
for (const name of ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1']) {
expect(await readFile(join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('promoted while queued');
}
});
it('keeps a provisioning failure separate while the other servers complete', async () => {
const { preview } = await previewLineup();
const { queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(komodoCreateDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'Host rejected this container' });
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
await processCreations();
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ queued: 4, failed: 1, done: 0 });
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
for (let pass = 0; pass < preview.servers.length; pass++) await processCreations();
const detail = getCollection(queued.collectionId!)!;
expect(detail.collection).toMatchObject({ total: 5, active: 0, done: 4, failed: 1 });
expect(detail.servers[0]).toMatchObject({ state: 'failed', error: expect.stringContaining('Host rejected') });
expect(detail.servers.slice(1).every(server => server.state === 'done')).toBe(true);
});
it('validates collection API payloads and rejects an unused expired preview', async () => {
const { preview, blueprint } = await previewLineup();
const { POST: singlePreview } = await import('../../src/routes/api/deployments/preview/+server.js');
const oldEndpoint = await singlePreview({ request: new Request('http://localhost/api/deployments/preview', {
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ blueprintId: blueprint.id })
}) });
expect(oldEndpoint.status).toBe(400);
expect((await oldEndpoint.json()).error).toContain('target counts');
const { db } = await import('../../src/lib/server/db/index.js');
const { queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const { POST } = await import('../../src/routes/api/deployment-collections/+server.js');
for (const body of [null, [], {}, { previewId: preview.id, keys: [] }, { previewId: preview.id, keys: { wrong: 'key' } }]) {
const response = await POST({ request: new Request('http://localhost/api/deployment-collections', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }) });
expect(response.status).toBe(400);
}
db().prepare("UPDATE server_collection_previews SET expires_at = '2000-01-01T00:00:00Z' WHERE id = ?").run(preview.id);
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/expired/);
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]);
});
});
describe('matching a blueprint exactly', () => {
async function blueprint(entries: { game: string; mapSlug: string; count: number }[]) {
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
return saveBlueprint({ name: 'Exact lineup', entries });
}
it('keeps the lowest deployed lobbies, removes excess and unrelated maps, and adds only the shortfall', async () => {
const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js');
for (const mapSlug of ['theater', 'theater', 'moon']) await createServer({ gameMode: 'zombie', mapSlug, plutoKey: 'existing' });
const { db } = await import('../../src/lib/server/db/index.js');
const before = db().prepare("SELECT * FROM servers WHERE name = 't5_zombie_theater_1'").get();
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }, { game: 't6', mapSlug: 'town', count: 1 }]);
const preview = previewCollection(plan.id, 'exact');
expect(preview.retainedServers).toEqual(['t5_zombie_theater_1']);
expect(preview.removals.map(server => server.name)).toEqual(['t5_zombie_moon_1', 't5_zombie_theater_2']);
expect(preview.servers.map(server => server.name)).toEqual(['t6_zombie_town_1']);
const { komodoDestroyDeployment, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(komodoCreateDeployment).mockClear();
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
expect(komodoCreateDeployment).not.toHaveBeenCalled();
expect(getCollection(result.collectionId!)?.collection).toMatchObject({ mode: 'exact', total: 3, active: 3, queued: 3 });
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }).mockResolvedValueOnce({ ok: false, reason: 'gone' });
await processDestructions();
await processCreations();
await processCreations();
expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: 3, done: 3, active: 0 });
expect(getCollection(result.collectionId!)?.servers.filter(server => server.action === 'destroy')).toHaveLength(2);
expect(db().prepare("SELECT * FROM servers WHERE name = 't5_zombie_theater_1'").get()).toEqual(before);
expect(db().prepare('SELECT name FROM servers ORDER BY name').all()).toEqual([{ name: 't5_zombie_theater_1' }, { name: 't6_zombie_town_1' }]);
expect(komodoDestroyDeployment).not.toHaveBeenCalledWith('t5_zombie_theater_1');
});
it('supports removal-only plans, shares existing removals, and records retained servers', async () => {
const { createServer } = await import('../../src/lib/server/fleet/create.js');
for (let i = 0; i < 2; i++) await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' });
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const first = previewCollection(plan.id, 'exact');
expect(first.servers).toEqual([]);
const a = queueCollection({ previewId: first.id });
const b = queueCollection({ previewId: previewCollection(plan.id, 'exact').id });
const firstDetail = getCollection(a.collectionId!)!;
expect(firstDetail.retainedServers).toEqual(['t5_zombie_theater_1']);
expect(firstDetail.servers).toHaveLength(1);
expect(getCollection(b.collectionId!)?.servers[0].id).toBe(firstDetail.servers[0].id);
expect(firstDetail.servers[0]).toMatchObject({ action: 'destroy', state: 'waiting', queuePosition: null });
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
expect(getDeployment(firstDetail.servers[0].id)?.collection).toBeTruthy();
});
it('cancels unstarted excess creations without booting them and retains their saved keys', async () => {
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
const first = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'first' });
const second = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'second' });
if (!first.ok || !second.ok) throw new Error('Fixture queue failed');
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const queued = queueCollection({ previewId: previewCollection(plan.id, 'exact').id });
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(second.deploymentId)).toEqual({ state: 'cancelled' });
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(first.deploymentId)).toEqual({ state: 'queued' });
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ done: 1, total: 1, active: 0 });
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
expect(getServerKey(second.name)).toBe('second');
const { komodoCreateDeployment, komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoCreateDeployment).not.toHaveBeenCalled();
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
});
it('rejects fleet changes even when the create shortfall stays the same', async () => {
const { createServer, queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' });
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const preview = previewCollection(plan.id, 'exact');
queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'new' });
expect(() => queueCollection({ previewId: preview.id })).toThrow(/fleet changed/);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
});
it('rolls back removals and creation cancellations when a required key is missing', async () => {
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
const old = queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
if (!old.ok) throw new Error(old.message);
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const preview = previewCollection(plan.id, 'exact');
expect(() => queueCollection({ previewId: preview.id })).toThrow(/No saved key/);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.deploymentId)).toEqual({ state: 'queued' });
for (const table of ['server_collection_removals', 'server_launch_collections', 'server_creation_dependencies']) expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
});
it('reuses a full fleet port only after empty-server removal, across restart and repeated submission', async () => {
vi.stubEnv('SERVER_PORT_MAX', '28960');
const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js');
await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
await processCreations();
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const preview = previewCollection(plan.id, 'exact');
expect(preview).toMatchObject({ ready: true, servers: [{ port: 28960 }] });
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
expect(queueCollection({ previewId: preview.id })).toEqual(result);
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoCreateDeployment, komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(komodoCreateDeployment).mockClear();
for (const players of [2, null]) {
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(players);
await processDestructions();
await processCreations();
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
expect(komodoCreateDeployment).not.toHaveBeenCalled();
expect(getCollection(result.collectionId!)?.servers.find(server => server.action === 'create')?.stage).toContain('Waiting for removal');
}
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' });
await processDestructions();
await processCreations();
await processCreations();
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: 2, done: 2, active: 0 });
});
it('does not reuse a port after a removal is cancelled', async () => {
vi.stubEnv('SERVER_PORT_MAX', '28960');
const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
await processCreations();
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const preview = previewCollection(plan.id, 'exact');
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js');
expect(cancelRemoval(getCollection(result.collectionId!)!.servers.find(server => server.action === 'destroy')!.id)).toBe(true);
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(komodoCreateDeployment).mockClear();
await processCreations();
expect(komodoCreateDeployment).not.toHaveBeenCalled();
expect(getCollection(result.collectionId!)?.collection.failed).toBe(1);
expect(getCollection(result.collectionId!)?.servers.find(server => server.action === 'create')?.stage).toContain('(cancelled)');
});
it('allows independent launches to pass a failed removal without exceeding the startup limit', async () => {
const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js');
await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
await processCreations();
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 2 }]);
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const preview = previewCollection(plan.id, 'exact');
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
const { komodoCreateDeployment, komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(komodoCreateDeployment).mockClear();
vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'Could not stop container' });
await processDestructions();
await processCreations();
const creates = getCollection(result.collectionId!)!.servers.filter(server => server.action === 'create');
expect(creates[0]).toMatchObject({ state: 'queued', stage: expect.stringContaining('(failed)') });
expect(creates[1]).toMatchObject({ state: 'awaiting-health' });
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
expect(getCollection(result.collectionId!)!.collection).toMatchObject({ failed: 1, queued: 1, verifying: 1 });
});
it('rejects insufficient final capacity before cancelling any existing creation', async () => {
vi.stubEnv('SERVER_PORT_MAX', '28960');
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
const old = queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
if (!old.ok) throw new Error(old.message);
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 2 }]);
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const preview = previewCollection(plan.id, 'exact');
expect(preview.ready).toBe(false);
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/not enough free game ports/);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.deploymentId)).toEqual({ state: 'queued' });
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
});
it('requires a new review when a pending removal is cancelled', async () => {
const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
const old = await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' });
if (!old.ok) throw new Error(old.message);
const removal = destroyServer(old.name);
if (!removal.ok) throw new Error(removal.message);
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const preview = previewCollection(plan.id, 'exact');
const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js');
cancelRemoval(removal.deploymentId!);
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/fleet changed/);
expect(previewCollection(plan.id, 'exact')).toMatchObject({ servers: [], removals: [], retainedServers: [old.name] });
});
it('keeps T6 variants distinct and leaves a matching fleet untouched', async () => {
const { createServer } = await import('../../src/lib/server/fleet/create.js');
await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'town' });
const plan = await blueprint([{ game: 't6', mapSlug: 'town', count: 1 }]);
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const noOp = previewCollection(plan.id, 'exact');
expect(noOp).toMatchObject({ retainedServers: ['t6_zombie_town_1'], removals: [], servers: [] });
expect(queueCollection({ previewId: noOp.id })).toEqual({ ok: true, collectionId: null });
await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'farm', plutoKey: 'farm' });
const preview = previewCollection(plan.id, 'exact');
expect(preview.retainedServers).toEqual(['t6_zombie_town_1']);
expect(preview.removals.map(server => server.name)).toEqual(['t6_zombie_farm_1']);
});
it('defaults old previews and collections to fill and validates the preview mode API', async () => {
const { blueprint: plan } = await previewLineup();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
db().exec('ALTER TABLE server_collection_previews DROP COLUMN mode');
db().exec('ALTER TABLE server_collection_previews DROP COLUMN fleet_snapshot');
db().exec('ALTER TABLE server_launch_collections DROP COLUMN mode');
// Recreate the older schema, which predates the history preservation trigger.
db().exec('DROP TRIGGER preserve_deleted_server_history');
db().exec('ALTER TABLE server_launch_collections DROP COLUMN retained_servers');
closeDb();
const { previewCollection } = await import('../../src/lib/server/fleet/collections.js');
expect(previewCollection(plan.id).mode).toBe('fill');
const { POST } = await import('../../src/routes/api/deployment-collections/preview/+server.js');
for (const mode of ['exact', 'fill', 'invalid', null, true]) {
const response = await POST({ request: new Request('http://localhost/api/deployment-collections/preview', { method: 'POST', body: JSON.stringify({ blueprintId: plan.id, mode }) }) });
expect(response.status).toBe(mode === 'exact' || mode === 'fill' ? 200 : 400);
if (response.ok) expect((await response.json()).mode).toBe(mode);
}
});
});