feat: match fleet to blueprint targets exactly
Add an optional exact blueprint mode with keep/create/remove previews, atomic queueing, cancellation of unstarted excess creations, and persistent dependencies for safe port reuse. Track removals alongside creations and preserve fill mode as the default. Validation: type check, production build, and 103 deployment tests passed. The full suite has one telemetry assertion failure also reproduced on the previous commit. Fixes #42
This commit is contained in:
@@ -359,3 +359,234 @@ describe('atomic collection launches', () => {
|
||||
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('matching a blueprint exactly', () => {
|
||||
async function blueprint(entries: { game: string; mapSlug: string; count: number }[]) {
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
return saveBlueprint({ name: 'Exact lineup', entries });
|
||||
}
|
||||
|
||||
it('keeps the lowest deployed lobbies, removes excess and unrelated maps, and adds only the shortfall', async () => {
|
||||
const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
for (const mapSlug of ['theater', 'theater', 'moon']) await createServer({ gameMode: 'zombie', mapSlug, plutoKey: 'existing' });
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const before = db().prepare("SELECT * FROM servers WHERE name = 't5_zombie_theater_1'").get();
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }, { game: 't6', mapSlug: 'town', count: 1 }]);
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
expect(preview.retainedServers).toEqual(['t5_zombie_theater_1']);
|
||||
expect(preview.removals.map(server => server.name)).toEqual(['t5_zombie_moon_1', 't5_zombie_theater_2']);
|
||||
expect(preview.servers.map(server => server.name)).toEqual(['t6_zombie_town_1']);
|
||||
const { komodoDestroyDeployment, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoCreateDeployment).mockClear();
|
||||
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
expect(getCollection(result.collectionId!)?.collection).toMatchObject({ mode: 'exact', total: 3, active: 3, queued: 3 });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' }).mockResolvedValueOnce({ ok: false, reason: 'gone' });
|
||||
await processDestructions();
|
||||
await processCreations();
|
||||
await processCreations();
|
||||
expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: 3, done: 3, active: 0 });
|
||||
expect(getCollection(result.collectionId!)?.servers.filter(server => server.action === 'destroy')).toHaveLength(2);
|
||||
expect(db().prepare("SELECT * FROM servers WHERE name = 't5_zombie_theater_1'").get()).toEqual(before);
|
||||
expect(db().prepare('SELECT name FROM servers ORDER BY name').all()).toEqual([{ name: 't5_zombie_theater_1' }, { name: 't6_zombie_town_1' }]);
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalledWith('t5_zombie_theater_1');
|
||||
});
|
||||
|
||||
it('supports removal-only plans, shares existing removals, and records retained servers', async () => {
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
for (let i = 0; i < 2; i++) await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' });
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const first = previewCollection(plan.id, 'exact');
|
||||
expect(first.servers).toEqual([]);
|
||||
const a = queueCollection({ previewId: first.id });
|
||||
const b = queueCollection({ previewId: previewCollection(plan.id, 'exact').id });
|
||||
const firstDetail = getCollection(a.collectionId!)!;
|
||||
expect(firstDetail.retainedServers).toEqual(['t5_zombie_theater_1']);
|
||||
expect(firstDetail.servers).toHaveLength(1);
|
||||
expect(getCollection(b.collectionId!)?.servers[0].id).toBe(firstDetail.servers[0].id);
|
||||
expect(firstDetail.servers[0]).toMatchObject({ action: 'destroy', state: 'waiting', queuePosition: null });
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
expect(getDeployment(firstDetail.servers[0].id)?.collection).toBeTruthy();
|
||||
});
|
||||
|
||||
it('cancels unstarted excess creations without booting them and retains their saved keys', async () => {
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const first = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'first' });
|
||||
const second = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'second' });
|
||||
if (!first.ok || !second.ok) throw new Error('Fixture queue failed');
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const queued = queueCollection({ previewId: previewCollection(plan.id, 'exact').id });
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(second.deploymentId)).toEqual({ state: 'cancelled' });
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(first.deploymentId)).toEqual({ state: 'queued' });
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ done: 1, total: 1, active: 0 });
|
||||
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
expect(getServerKey(second.name)).toBe('second');
|
||||
const { komodoCreateDeployment, komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects fleet changes even when the create shortfall stays the same', async () => {
|
||||
const { createServer, queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' });
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'new' });
|
||||
expect(() => queueCollection({ previewId: preview.id })).toThrow(/fleet changed/);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('rolls back removals and creation cancellations when a required key is missing', async () => {
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const old = queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
|
||||
if (!old.ok) throw new Error(old.message);
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
expect(() => queueCollection({ previewId: preview.id })).toThrow(/No saved key/);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.deploymentId)).toEqual({ state: 'queued' });
|
||||
for (const table of ['server_collection_removals', 'server_launch_collections', 'server_creation_dependencies']) expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
|
||||
expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('reuses a full fleet port only after empty-server removal, across restart and repeated submission', async () => {
|
||||
vi.stubEnv('SERVER_PORT_MAX', '28960');
|
||||
const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
|
||||
await processCreations();
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
expect(preview).toMatchObject({ ready: true, servers: [{ port: 28960 }] });
|
||||
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
expect(queueCollection({ previewId: preview.id })).toEqual(result);
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoCreateDeployment, komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoCreateDeployment).mockClear();
|
||||
for (const players of [2, null]) {
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(players);
|
||||
await processDestructions();
|
||||
await processCreations();
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
expect(getCollection(result.collectionId!)?.servers.find(server => server.action === 'create')?.stage).toContain('Waiting for removal');
|
||||
}
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'gone' });
|
||||
await processDestructions();
|
||||
await processCreations();
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: 2, done: 2, active: 0 });
|
||||
});
|
||||
|
||||
it('does not reuse a port after a removal is cancelled', async () => {
|
||||
vi.stubEnv('SERVER_PORT_MAX', '28960');
|
||||
const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
|
||||
await processCreations();
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
expect(cancelRemoval(getCollection(result.collectionId!)!.servers.find(server => server.action === 'destroy')!.id)).toBe(true);
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoCreateDeployment).mockClear();
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
expect(getCollection(result.collectionId!)?.collection.failed).toBe(1);
|
||||
expect(getCollection(result.collectionId!)?.servers.find(server => server.action === 'create')?.stage).toContain('(cancelled)');
|
||||
});
|
||||
|
||||
it('allows independent launches to pass a failed removal without exceeding the startup limit', async () => {
|
||||
const { createServer, processCreations, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
await createServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
|
||||
await processCreations();
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 2 }]);
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
const result = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
const { komodoCreateDeployment, komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoCreateDeployment).mockClear();
|
||||
vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'Could not stop container' });
|
||||
await processDestructions();
|
||||
await processCreations();
|
||||
const creates = getCollection(result.collectionId!)!.servers.filter(server => server.action === 'create');
|
||||
expect(creates[0]).toMatchObject({ state: 'queued', stage: expect.stringContaining('(failed)') });
|
||||
expect(creates[1]).toMatchObject({ state: 'awaiting-health' });
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(getCollection(result.collectionId!)!.collection).toMatchObject({ failed: 1, queued: 1, verifying: 1 });
|
||||
});
|
||||
|
||||
it('rejects insufficient final capacity before cancelling any existing creation', async () => {
|
||||
vi.stubEnv('SERVER_PORT_MAX', '28960');
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const old = queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
|
||||
if (!old.ok) throw new Error(old.message);
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 2 }]);
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
expect(preview.ready).toBe(false);
|
||||
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/not enough free game ports/);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.deploymentId)).toEqual({ state: 'queued' });
|
||||
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('requires a new review when a pending removal is cancelled', async () => {
|
||||
const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const old = await createServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'existing' });
|
||||
if (!old.ok) throw new Error(old.message);
|
||||
const removal = destroyServer(old.name);
|
||||
if (!removal.ok) throw new Error(removal.message);
|
||||
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
cancelRemoval(removal.deploymentId!);
|
||||
expect(() => queueCollection({ previewId: preview.id, keys: keysFor(preview) })).toThrow(/fleet changed/);
|
||||
expect(previewCollection(plan.id, 'exact')).toMatchObject({ servers: [], removals: [], retainedServers: [old.name] });
|
||||
});
|
||||
|
||||
it('keeps T6 variants distinct and leaves a matching fleet untouched', async () => {
|
||||
const { createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town', plutoKey: 'town' });
|
||||
const plan = await blueprint([{ game: 't6', mapSlug: 'town', count: 1 }]);
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const noOp = previewCollection(plan.id, 'exact');
|
||||
expect(noOp).toMatchObject({ retainedServers: ['t6_zombie_town_1'], removals: [], servers: [] });
|
||||
expect(queueCollection({ previewId: noOp.id })).toEqual({ ok: true, collectionId: null });
|
||||
await createServer({ game: 't6', gameMode: 'zombie', mapSlug: 'farm', plutoKey: 'farm' });
|
||||
const preview = previewCollection(plan.id, 'exact');
|
||||
expect(preview.retainedServers).toEqual(['t6_zombie_town_1']);
|
||||
expect(preview.removals.map(server => server.name)).toEqual(['t6_zombie_farm_1']);
|
||||
});
|
||||
|
||||
it('defaults old previews and collections to fill and validates the preview mode API', async () => {
|
||||
const { blueprint: plan } = await previewLineup();
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
db().exec('ALTER TABLE server_collection_previews DROP COLUMN mode');
|
||||
db().exec('ALTER TABLE server_collection_previews DROP COLUMN fleet_snapshot');
|
||||
db().exec('ALTER TABLE server_launch_collections DROP COLUMN mode');
|
||||
db().exec('ALTER TABLE server_launch_collections DROP COLUMN retained_servers');
|
||||
closeDb();
|
||||
const { previewCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
expect(previewCollection(plan.id).mode).toBe('fill');
|
||||
const { POST } = await import('../../src/routes/api/deployment-collections/preview/+server.js');
|
||||
for (const mode of ['exact', 'fill', 'invalid', null, true]) {
|
||||
const response = await POST({ request: new Request('http://localhost/api/deployment-collections/preview', { method: 'POST', body: JSON.stringify({ blueprintId: plan.id, mode }) }) });
|
||||
expect(response.status).toBe(mode === 'exact' || mode === 'fill' ? 200 : 400);
|
||||
if (response.ok) expect((await response.json()).mode).toBe(mode);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user