358 lines
22 KiB
TypeScript
358 lines
22 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { existsSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
|
komodoCreateDeployment: vi.fn(),
|
|
komodoDeploy: vi.fn(),
|
|
komodoDestroyDeployment: vi.fn(),
|
|
komodoDeleteDeployment: vi.fn(),
|
|
komodoInspectHealthy: vi.fn(),
|
|
komodoEnsureTag: vi.fn(),
|
|
komodoUpdateTagsOnResource: vi.fn()
|
|
}));
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
|
rconGetPlayerCount: vi.fn(),
|
|
rconStatusResult: vi.fn()
|
|
}));
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
|
|
|
let fixture: Fixture;
|
|
const input = { gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'test-key' };
|
|
function status(server: string, map = server.includes('_moon_') ? 'zombie_moon' : 'zombie_theater') {
|
|
return { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: '',
|
|
status: { players: 0, map, gametype: null, roster: [] } };
|
|
}
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules(); vi.resetAllMocks();
|
|
fixture = await newFixture([]);
|
|
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
|
vi.stubEnv('KOMODO_API_KEY', 'test-key');
|
|
vi.stubEnv('KOMODO_API_SECRET', 'test-secret');
|
|
vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host');
|
|
vi.stubEnv('RCON_PASSWORD', 'test-password');
|
|
vi.stubEnv('SERVER_PORT_MIN', '28960');
|
|
vi.stubEnv('SERVER_PORT_MAX', '28980');
|
|
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1');
|
|
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodo.komodoCreateDeployment).mockResolvedValue({ ok: true });
|
|
vi.mocked(komodo.komodoDeploy).mockResolvedValue({ ok: true });
|
|
vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: true });
|
|
vi.mocked(komodo.komodoDeleteDeployment).mockResolvedValue({ ok: true });
|
|
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
|
|
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
|
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
|
vi.mocked(rcon.rconStatusResult).mockImplementation(async server => status(server));
|
|
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
|
|
vi.mocked(kick).mockResolvedValue(undefined);
|
|
});
|
|
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); });
|
|
|
|
async function queueServers(count: number) {
|
|
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
|
return Array.from({ length: count }, () => {
|
|
const result = queueCreateServer(input);
|
|
if (!result.ok) throw new Error(result.message);
|
|
return result;
|
|
});
|
|
}
|
|
async function states() {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
return (db().prepare("SELECT state FROM server_deployments WHERE action = 'create' ORDER BY rowid").all() as { state: string }[]).map(row => row.state);
|
|
}
|
|
|
|
describe('shared creation queue', () => {
|
|
it('starts only one server per pass and holds its slot until a later health check', async () => {
|
|
const jobs = await queueServers(3);
|
|
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await processCreations();
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
|
expect(komodoInspectHealthy).not.toHaveBeenCalled();
|
|
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']);
|
|
await processCreations();
|
|
expect(komodoInspectHealthy).toHaveBeenCalledWith(jobs[0].name);
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
|
expect(await states()).toEqual(['done', 'awaiting-health', 'queued']);
|
|
expect(new Set(jobs.map(job => job.port)).size).toBe(3);
|
|
});
|
|
|
|
it('keeps later boots queued through container, RCON, and expected-map failures', async () => {
|
|
const jobs = await queueServers(2);
|
|
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
|
await processCreations();
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' });
|
|
await processCreations();
|
|
expect(rconStatusResult).not.toHaveBeenCalled();
|
|
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for Komodo health: starting' });
|
|
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, reason: 'timeout', endpoint: null, detail: 'booting' });
|
|
await processCreations();
|
|
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for RCON or fresh telemetry health: timeout' });
|
|
vi.mocked(rconStatusResult).mockResolvedValueOnce(status(jobs[0].name, 'zombie_moon'));
|
|
await processCreations();
|
|
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: expect.stringContaining('expected zombie_theater') });
|
|
vi.mocked(komodoInspectHealthy).mockRejectedValueOnce(new Error('Temporary connection error'));
|
|
await processCreations();
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
|
expect(await states()).toEqual(['awaiting-health', 'queued']);
|
|
await processCreations();
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
|
expect(await states()).toEqual(['done', 'awaiting-health']);
|
|
});
|
|
|
|
it('shares FIFO capacity across collections and direct callers, including equal timestamps', async () => {
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
|
const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const collections = ['theater', 'moon'].map(mapSlug => {
|
|
const blueprint = saveBlueprint({ name: mapSlug, entries: [{ game: 't5', mapSlug, count: 2 }] });
|
|
const preview = previewCollection(blueprint.id);
|
|
return queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name, 'test-key'])) });
|
|
});
|
|
db().prepare("UPDATE server_deployments SET started_at = '2026-01-01T00:00:00.000Z'").run();
|
|
const direct = await createServer(input);
|
|
if (!direct.ok) throw new Error(direct.message);
|
|
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued', 'queued', 'queued']);
|
|
const second = getCollection(collections[1].collectionId!)!;
|
|
expect(second.collection).toMatchObject({ queued: 2, provisioning: 0, verifying: 0 });
|
|
expect(second.queue).toMatchObject({ concurrency: 1, queued: 4, active: [{ server: 't5_zombie_theater_1' }] });
|
|
expect(second.servers.map(server => server.queuePosition)).toEqual([2, 3]);
|
|
for (let pass = 0; pass < 5; pass++) await processCreations();
|
|
expect(vi.mocked(komodoCreateDeployment).mock.calls.map(([spec]) => spec.name)).toEqual([
|
|
't5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1', 't5_zombie_moon_2', direct.name
|
|
]);
|
|
expect(await states()).toEqual(['done', 'done', 'done', 'done', 'done']);
|
|
});
|
|
|
|
it('coalesces overlapping worker passes and direct calls while provisioning is pending', async () => {
|
|
await queueServers(3);
|
|
const { processCreations, createServer } = await import('../../src/lib/server/fleet/create.js');
|
|
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
|
let release!: () => void;
|
|
let entered!: () => void;
|
|
const blocked = new Promise<void>(resolve => { release = resolve; });
|
|
const started = new Promise<void>(resolve => { entered = resolve; });
|
|
vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; });
|
|
const first = processCreations();
|
|
await started;
|
|
const second = processCreations();
|
|
const direct = createServer(input);
|
|
expect(await states()).toEqual(['running', 'queued', 'queued', 'queued']);
|
|
release();
|
|
await Promise.all([first, second, direct]);
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
|
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued', 'queued']);
|
|
});
|
|
|
|
it('honors a higher limit without refilling its pass budget after immediate failures', async () => {
|
|
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
|
|
await queueServers(5);
|
|
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodoCreateDeployment).mockResolvedValue({ ok: false, kind: 'http-error', message: 'Host unavailable' });
|
|
await processCreations();
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
|
expect(await states()).toEqual(['failed', 'failed', 'queued', 'queued', 'queued']);
|
|
vi.mocked(komodoCreateDeployment).mockResolvedValue({ ok: true });
|
|
await processCreations();
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(4);
|
|
expect(await states()).toEqual(['failed', 'failed', 'awaiting-health', 'awaiting-health', 'queued']);
|
|
});
|
|
|
|
it('does not provision a selected job that was cancelled during an earlier startup', async () => {
|
|
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
|
|
const jobs = await queueServers(3);
|
|
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { finishDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
|
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
|
let release!: () => void;
|
|
let entered!: () => void;
|
|
const blocked = new Promise<void>(resolve => { release = resolve; });
|
|
const started = new Promise<void>(resolve => { entered = resolve; });
|
|
vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; });
|
|
const pass = processCreations();
|
|
await started;
|
|
finishDeployment(jobs[1].deploymentId, { ok: false, message: 'Cancelled before startup' });
|
|
release();
|
|
await pass;
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
|
expect(await states()).toEqual(['awaiting-health', 'failed', 'queued']);
|
|
});
|
|
|
|
it('rechecks capacity when an existing server resumes verification during provisioning', async () => {
|
|
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
|
|
const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { finishDeployment, resumeVerification } = await import('../../src/lib/server/fleet/deployments.js');
|
|
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const old = await createServer(input);
|
|
if (!old.ok) throw new Error(old.message);
|
|
finishDeployment(old.deploymentId, { ok: false, message: 'Needs verification' });
|
|
await queueServers(2);
|
|
let release!: () => void;
|
|
let entered!: () => void;
|
|
const blocked = new Promise<void>(resolve => { release = resolve; });
|
|
const started = new Promise<void>(resolve => { entered = resolve; });
|
|
vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; });
|
|
const pass = processCreations();
|
|
await started;
|
|
expect(resumeVerification(old.deploymentId)).toBe(true);
|
|
release();
|
|
await pass;
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
|
expect(await states()).toEqual(['awaiting-health', 'awaiting-health', 'queued']);
|
|
});
|
|
|
|
it('counts already verifying servers against a higher concurrency limit', async () => {
|
|
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
|
|
const jobs = await queueServers(4);
|
|
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await processCreations();
|
|
vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === jobs[0].name
|
|
? { ok: false, reason: 'still booting' } : { ok: true, state: 'running' });
|
|
await processCreations();
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(3);
|
|
expect(await states()).toEqual(['awaiting-health', 'done', 'awaiting-health', 'queued']);
|
|
});
|
|
|
|
it('retains durable reservations and the startup gate across an application restart', async () => {
|
|
const jobs = await queueServers(3);
|
|
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { komodoCreateDeployment, komodoDeploy, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await processCreations();
|
|
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(jobs[0].deploymentId);
|
|
closeDb();
|
|
vi.resetModules();
|
|
const { interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
|
|
const { processCreations: resumed, nextAvailablePort } = await import('../../src/lib/server/fleet/create.js');
|
|
interruptDeployments();
|
|
expect(await states()).toEqual(['interrupted', 'queued', 'queued']);
|
|
expect(nextAvailablePort()).toBe(28963);
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' });
|
|
await resumed();
|
|
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']);
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
|
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
|
await resumed();
|
|
expect(await states()).toEqual(['done', 'awaiting-health', 'queued']);
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('keeps an ambiguous startup response in verification without replaying its deployment', async () => {
|
|
await queueServers(2);
|
|
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { komodoDeploy, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodoDeploy).mockRejectedValueOnce(new Error('Response lost after starting'));
|
|
const log = vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
await processCreations();
|
|
expect(await states()).toEqual(['awaiting-health', 'queued']);
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' });
|
|
await processCreations();
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
|
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
|
expect(log).toHaveBeenCalled();
|
|
await processCreations();
|
|
expect(await states()).toEqual(['done', 'awaiting-health']);
|
|
});
|
|
|
|
it.each([
|
|
{ ok: false as const, kind: 'unreachable' as const, message: 'Request timed out after Komodo accepted it' },
|
|
{ ok: false as const, kind: 'http-error' as const, httpStatus: 502, message: 'Gateway lost the startup response' }
|
|
])('holds the slot and files for an uncertain structured response: $kind', async failure => {
|
|
const jobs = await queueServers(2);
|
|
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
|
const { komodoDeploy, komodoDeleteDeployment, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodoDeploy).mockResolvedValueOnce(failure);
|
|
await processCreations();
|
|
expect(await states()).toEqual(['awaiting-health', 'queued']);
|
|
expect(getServer(jobs[0].name)?.game_port).toBe(jobs[0].port);
|
|
expect(existsSync(join(fixture.t5, jobs[0].name, 'configs/electric_cherry_map.cfg'))).toBe(true);
|
|
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
|
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' });
|
|
await processCreations();
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
|
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
|
await processCreations();
|
|
expect(await states()).toEqual(['done', 'awaiting-health']);
|
|
});
|
|
|
|
it('releases capacity on a confirmed deployment rejection', async () => {
|
|
const jobs = await queueServers(2);
|
|
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
|
const { komodoDeploy, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', httpStatus: 403, message: 'Access denied' });
|
|
await processCreations();
|
|
expect(await states()).toEqual(['failed', 'queued']);
|
|
expect(getServer(jobs[0].name)).toBeUndefined();
|
|
expect(existsSync(join(fixture.t5, jobs[0].name))).toBe(false);
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
|
await processCreations();
|
|
expect(await states()).toEqual(['failed', 'awaiting-health']);
|
|
});
|
|
|
|
it('retains an interrupted startup slot while a newer destruction is still waiting', async () => {
|
|
const jobs = await queueServers(2);
|
|
const { processCreations, destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
|
const { interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await processCreations();
|
|
expect(destroyServer(jobs[0].name)).toMatchObject({ ok: true });
|
|
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(jobs[0].deploymentId);
|
|
interruptDeployments();
|
|
expect(await states()).toEqual(['interrupted', 'queued']);
|
|
await processCreations();
|
|
expect(await states()).toEqual(['awaiting-health', 'queued']);
|
|
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
|
expect(komodoInspectHealthy).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('releases a startup slot after its server is removed', async () => {
|
|
const jobs = await queueServers(2);
|
|
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
await processCreations();
|
|
db().prepare('DELETE FROM servers WHERE name = ?').run(jobs[0].name);
|
|
await processCreations();
|
|
expect(await states()).toEqual(['failed', 'awaiting-health']);
|
|
});
|
|
|
|
it('never verifies a replacement registered under the same server name', async () => {
|
|
const jobs = await queueServers(2);
|
|
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
await processCreations();
|
|
db().prepare('UPDATE servers SET game_port = ? WHERE name = ?').run(28980, jobs[0].name);
|
|
await processCreations();
|
|
expect(komodoInspectHealthy).not.toHaveBeenCalledWith(jobs[0].name);
|
|
expect(await states()).toEqual(['failed', 'awaiting-health']);
|
|
});
|
|
});
|
|
|
|
describe('creation concurrency configuration', () => {
|
|
it('defaults to one startup', async () => {
|
|
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', undefined);
|
|
const { config } = await import('../../src/lib/server/config.js');
|
|
expect(config.createConcurrency).toBe(1);
|
|
});
|
|
|
|
it.each(['0', '-1', '1.5', 'invalid', 'Infinity', '9007199254740992'])('rejects invalid limit %s', async value => {
|
|
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', value);
|
|
await expect(import('../../src/lib/server/config.js')).rejects.toThrow(/DEPLOYMENT_CREATE_CONCURRENCY/);
|
|
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1');
|
|
});
|
|
});
|