Pace collection startups and clarify deployment queue progress
This commit is contained in:
@@ -34,6 +34,13 @@ beforeEach(async () => {
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'test');
|
||||
vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host');
|
||||
vi.stubEnv('RCON_PASSWORD', 'test-rcon');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult, rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(komodoInspectHealthy).mockReset().mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(rconStatusResult).mockReset().mockResolvedValue({ ok: true,
|
||||
endpoint: { host: 'localhost', port: 27016 }, rawReply: '',
|
||||
status: { players: 0, map: 'zombie_cosmodrome', gametype: null, roster: [] } });
|
||||
vi.mocked(rconGetPlayerCount).mockReset().mockResolvedValue(0);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -48,7 +55,10 @@ describe('destruction cleanup recovery', () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true,
|
||||
endpoint: { host: 'localhost', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map: 'zm_transit', gametype: null, roster: [] } });
|
||||
const name = 't6_zombie_town_1';
|
||||
expect(await createServer({ ...input, game: 't6', mapSlug: 'town' })).toMatchObject({ ok: true, name });
|
||||
expect(await createServer(input)).toMatchObject({ ok: true });
|
||||
@@ -61,6 +71,7 @@ describe('destruction cleanup recovery', () => {
|
||||
db().prepare("INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, 'test.gsc')").run(beta, server);
|
||||
}
|
||||
const queued = destroyServer(name);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
|
||||
await processDestructions();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
|
||||
for (const table of ['beta_test_runs', 'beta_test_coverage', 'beta_file_changes']) {
|
||||
@@ -269,6 +280,12 @@ describe('T6 Zombies provisioning', () => {
|
||||
vi.mocked(rconStatusResult).mockImplementation(async (server) => ({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map: server.startsWith('t6_') ? 'zm_transit' : 'zombie_cosmodrome', gametype: null, roster: [] } }));
|
||||
await processCreations();
|
||||
expect(getDeployment(t5.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' });
|
||||
expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'queued' });
|
||||
await processCreations();
|
||||
expect(getDeployment(t5.deploymentId)?.deployment).toMatchObject({ state: 'done' });
|
||||
expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' });
|
||||
await processCreations();
|
||||
expect(getDeployment(t6.deploymentId)?.deployment).toMatchObject({ state: 'done' });
|
||||
const content = await readFile(join(fixture.t6, t6.name, 'configs/electric_cherry_map.cfg'), 'utf8');
|
||||
expect(content).toContain('execgts zm_standard_town.cfg map zm_transit');
|
||||
@@ -340,10 +357,13 @@ describe('server creation map selection', () => {
|
||||
closeDb();
|
||||
interruptDeployments();
|
||||
expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'queued' });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' }).mockResolvedValueOnce({ ok: false, reason: 'starting' });
|
||||
await Promise.all([processCreations(), processCreations()]);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(getDeployment(body.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null });
|
||||
if (second.ok) expect(getDeployment(second.deploymentId)?.deployment).toMatchObject({ state: 'queued' });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('completes creation only when Komodo, RCON, and the selected map all pass', async () => {
|
||||
@@ -469,7 +489,8 @@ describe('server creation map selection', () => {
|
||||
expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'failed', error: 'Deploy: test failure' }));
|
||||
vi.mocked(komodoDeploy).mockRejectedValueOnce(new Error('connection lost'));
|
||||
await expect(createServer(input)).rejects.toThrow('connection lost');
|
||||
expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'failed', error: 'connection lost' }));
|
||||
expect(listDeployments()).toContainEqual(expect.objectContaining({ state: 'awaiting-health',
|
||||
stage: 'Verifying existing server after interrupted startup', finished_at: null }));
|
||||
const id = beginDeployment('interrupted-server', 'destroy');
|
||||
interruptDeployments();
|
||||
expect(listDeployments()).toContainEqual(expect.objectContaining({ id, state: 'interrupted', finished_at: expect.any(String) }));
|
||||
@@ -587,12 +608,12 @@ describe('persistent server keys', () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getServerKey, listServerKeys } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, kind: 'not-found', message: 'gone' } as never);
|
||||
const name = 't5_zombie_cosmodrome_2';
|
||||
expect(await createServer(input)).toMatchObject({ ok: true, name: serverName });
|
||||
expect(await createServer(input)).toMatchObject({ ok: true, name });
|
||||
expect(getServerKey(name)).toBe('test-key');
|
||||
expect(await destroyServer(name)).toMatchObject({ ok: true });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
|
||||
await processDestructions();
|
||||
expect(listServerKeys()).toContainEqual(expect.objectContaining({ server_id: name, deployed: 0 }));
|
||||
expect(JSON.stringify(listServerKeys())).not.toContain('test-key');
|
||||
@@ -621,10 +642,10 @@ describe('persistent server keys', () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false } as never);
|
||||
expect(await createServer(input)).toMatchObject({ ok: true, name: serverName });
|
||||
expect(await createServer(input)).toMatchObject({ ok: true, name: 't5_zombie_cosmodrome_2' });
|
||||
await destroyServer(serverName);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
|
||||
await processDestructions();
|
||||
expect(await createServer({ ...input, plutoKey: 'wrong-key', expectedName: 't5_zombie_cosmodrome_3' }))
|
||||
.toMatchObject({ ok: false, kind: 'no-slot' });
|
||||
|
||||
@@ -0,0 +1,357 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoCreateDeployment: vi.fn(),
|
||||
komodoDeploy: vi.fn(),
|
||||
komodoDestroyDeployment: vi.fn(),
|
||||
komodoDeleteDeployment: vi.fn(),
|
||||
komodoInspectHealthy: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(),
|
||||
komodoUpdateTagsOnResource: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(),
|
||||
rconStatusResult: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
const input = { gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'test-key' };
|
||||
function status(server: string, map = server.includes('_moon_') ? 'zombie_moon' : 'zombie_theater') {
|
||||
return { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map, gametype: null, roster: [] } };
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.resetAllMocks();
|
||||
fixture = await newFixture([]);
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'test-key');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'test-secret');
|
||||
vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host');
|
||||
vi.stubEnv('RCON_PASSWORD', 'test-password');
|
||||
vi.stubEnv('SERVER_PORT_MIN', '28960');
|
||||
vi.stubEnv('SERVER_PORT_MAX', '28980');
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1');
|
||||
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodo.komodoCreateDeployment).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoDeploy).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoDestroyDeployment).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoDeleteDeployment).mockResolvedValue({ ok: true });
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
|
||||
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
||||
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconStatusResult).mockImplementation(async server => status(server));
|
||||
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
|
||||
vi.mocked(kick).mockResolvedValue(undefined);
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); });
|
||||
|
||||
async function queueServers(count: number) {
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
return Array.from({ length: count }, () => {
|
||||
const result = queueCreateServer(input);
|
||||
if (!result.ok) throw new Error(result.message);
|
||||
return result;
|
||||
});
|
||||
}
|
||||
async function states() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
return (db().prepare("SELECT state FROM server_deployments WHERE action = 'create' ORDER BY rowid").all() as { state: string }[]).map(row => row.state);
|
||||
}
|
||||
|
||||
describe('shared creation queue', () => {
|
||||
it('starts only one server per pass and holds its slot until a later health check', async () => {
|
||||
const jobs = await queueServers(3);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']);
|
||||
await processCreations();
|
||||
expect(komodoInspectHealthy).toHaveBeenCalledWith(jobs[0].name);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(await states()).toEqual(['done', 'awaiting-health', 'queued']);
|
||||
expect(new Set(jobs.map(job => job.port)).size).toBe(3);
|
||||
});
|
||||
|
||||
it('keeps later boots queued through container, RCON, and expected-map failures', async () => {
|
||||
const jobs = await queueServers(2);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
await processCreations();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' });
|
||||
await processCreations();
|
||||
expect(rconStatusResult).not.toHaveBeenCalled();
|
||||
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for Komodo health: starting' });
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, reason: 'timeout', endpoint: null, detail: 'booting' });
|
||||
await processCreations();
|
||||
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: 'Waiting for RCON health: timeout' });
|
||||
vi.mocked(rconStatusResult).mockResolvedValueOnce(status(jobs[0].name, 'zombie_moon'));
|
||||
await processCreations();
|
||||
expect(getDeployment(jobs[0].deploymentId)?.deployment).toMatchObject({ stage: expect.stringContaining('expected zombie_theater') });
|
||||
vi.mocked(komodoInspectHealthy).mockRejectedValueOnce(new Error('Temporary connection error'));
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued']);
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(await states()).toEqual(['done', 'awaiting-health']);
|
||||
});
|
||||
|
||||
it('shares FIFO capacity across collections and direct callers, including equal timestamps', async () => {
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const collections = ['theater', 'moon'].map(mapSlug => {
|
||||
const blueprint = saveBlueprint({ name: mapSlug, entries: [{ game: 't5', mapSlug, count: 2 }] });
|
||||
const preview = previewCollection(blueprint.id);
|
||||
return queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name, 'test-key'])) });
|
||||
});
|
||||
db().prepare("UPDATE server_deployments SET started_at = '2026-01-01T00:00:00.000Z'").run();
|
||||
const direct = await createServer(input);
|
||||
if (!direct.ok) throw new Error(direct.message);
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued', 'queued', 'queued']);
|
||||
const second = getCollection(collections[1].collectionId!)!;
|
||||
expect(second.collection).toMatchObject({ queued: 2, provisioning: 0, verifying: 0 });
|
||||
expect(second.queue).toMatchObject({ concurrency: 1, queued: 4, active: [{ server: 't5_zombie_theater_1' }] });
|
||||
expect(second.servers.map(server => server.queuePosition)).toEqual([2, 3]);
|
||||
for (let pass = 0; pass < 5; pass++) await processCreations();
|
||||
expect(vi.mocked(komodoCreateDeployment).mock.calls.map(([spec]) => spec.name)).toEqual([
|
||||
't5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1', 't5_zombie_moon_2', direct.name
|
||||
]);
|
||||
expect(await states()).toEqual(['done', 'done', 'done', 'done', 'done']);
|
||||
});
|
||||
|
||||
it('coalesces overlapping worker passes and direct calls while provisioning is pending', async () => {
|
||||
await queueServers(3);
|
||||
const { processCreations, createServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
let release!: () => void;
|
||||
let entered!: () => void;
|
||||
const blocked = new Promise<void>(resolve => { release = resolve; });
|
||||
const started = new Promise<void>(resolve => { entered = resolve; });
|
||||
vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; });
|
||||
const first = processCreations();
|
||||
await started;
|
||||
const second = processCreations();
|
||||
const direct = createServer(input);
|
||||
expect(await states()).toEqual(['running', 'queued', 'queued', 'queued']);
|
||||
release();
|
||||
await Promise.all([first, second, direct]);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued', 'queued']);
|
||||
});
|
||||
|
||||
it('honors a higher limit without refilling its pass budget after immediate failures', async () => {
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
|
||||
await queueServers(5);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoCreateDeployment).mockResolvedValue({ ok: false, kind: 'http-error', message: 'Host unavailable' });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(await states()).toEqual(['failed', 'failed', 'queued', 'queued', 'queued']);
|
||||
vi.mocked(komodoCreateDeployment).mockResolvedValue({ ok: true });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(4);
|
||||
expect(await states()).toEqual(['failed', 'failed', 'awaiting-health', 'awaiting-health', 'queued']);
|
||||
});
|
||||
|
||||
it('does not provision a selected job that was cancelled during an earlier startup', async () => {
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
|
||||
const jobs = await queueServers(3);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { finishDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
let release!: () => void;
|
||||
let entered!: () => void;
|
||||
const blocked = new Promise<void>(resolve => { release = resolve; });
|
||||
const started = new Promise<void>(resolve => { entered = resolve; });
|
||||
vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; });
|
||||
const pass = processCreations();
|
||||
await started;
|
||||
finishDeployment(jobs[1].deploymentId, { ok: false, message: 'Cancelled before startup' });
|
||||
release();
|
||||
await pass;
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(await states()).toEqual(['awaiting-health', 'failed', 'queued']);
|
||||
});
|
||||
|
||||
it('rechecks capacity when an existing server resumes verification during provisioning', async () => {
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
|
||||
const { createServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { finishDeployment, resumeVerification } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const old = await createServer(input);
|
||||
if (!old.ok) throw new Error(old.message);
|
||||
finishDeployment(old.deploymentId, { ok: false, message: 'Needs verification' });
|
||||
await queueServers(2);
|
||||
let release!: () => void;
|
||||
let entered!: () => void;
|
||||
const blocked = new Promise<void>(resolve => { release = resolve; });
|
||||
const started = new Promise<void>(resolve => { entered = resolve; });
|
||||
vi.mocked(komodoCreateDeployment).mockImplementationOnce(async () => { entered(); await blocked; return { ok: true }; });
|
||||
const pass = processCreations();
|
||||
await started;
|
||||
expect(resumeVerification(old.deploymentId)).toBe(true);
|
||||
release();
|
||||
await pass;
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(await states()).toEqual(['awaiting-health', 'awaiting-health', 'queued']);
|
||||
});
|
||||
|
||||
it('counts already verifying servers against a higher concurrency limit', async () => {
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
|
||||
const jobs = await queueServers(4);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await processCreations();
|
||||
vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === jobs[0].name
|
||||
? { ok: false, reason: 'still booting' } : { ok: true, state: 'running' });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(3);
|
||||
expect(await states()).toEqual(['awaiting-health', 'done', 'awaiting-health', 'queued']);
|
||||
});
|
||||
|
||||
it('retains durable reservations and the startup gate across an application restart', async () => {
|
||||
const jobs = await queueServers(3);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoCreateDeployment, komodoDeploy, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await processCreations();
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(jobs[0].deploymentId);
|
||||
closeDb();
|
||||
vi.resetModules();
|
||||
const { interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { processCreations: resumed, nextAvailablePort } = await import('../../src/lib/server/fleet/create.js');
|
||||
interruptDeployments();
|
||||
expect(await states()).toEqual(['interrupted', 'queued', 'queued']);
|
||||
expect(nextAvailablePort()).toBe(28963);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' });
|
||||
await resumed();
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued', 'queued']);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
await resumed();
|
||||
expect(await states()).toEqual(['done', 'awaiting-health', 'queued']);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('keeps an ambiguous startup response in verification without replaying its deployment', async () => {
|
||||
await queueServers(2);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDeploy, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoDeploy).mockRejectedValueOnce(new Error('Response lost after starting'));
|
||||
const log = vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued']);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
expect(log).toHaveBeenCalled();
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['done', 'awaiting-health']);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ ok: false as const, kind: 'unreachable' as const, message: 'Request timed out after Komodo accepted it' },
|
||||
{ ok: false as const, kind: 'http-error' as const, httpStatus: 502, message: 'Gateway lost the startup response' }
|
||||
])('holds the slot and files for an uncertain structured response: $kind', async failure => {
|
||||
const jobs = await queueServers(2);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { komodoDeploy, komodoDeleteDeployment, komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoDeploy).mockResolvedValueOnce(failure);
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued']);
|
||||
expect(getServer(jobs[0].name)?.game_port).toBe(jobs[0].port);
|
||||
expect(existsSync(join(fixture.t5, jobs[0].name, 'configs/electric_cherry_map.cfg'))).toBe(true);
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'still booting' });
|
||||
await processCreations();
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['done', 'awaiting-health']);
|
||||
});
|
||||
|
||||
it('releases capacity on a confirmed deployment rejection', async () => {
|
||||
const jobs = await queueServers(2);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { komodoDeploy, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoDeploy).mockResolvedValueOnce({ ok: false, kind: 'http-error', httpStatus: 403, message: 'Access denied' });
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['failed', 'queued']);
|
||||
expect(getServer(jobs[0].name)).toBeUndefined();
|
||||
expect(existsSync(join(fixture.t5, jobs[0].name))).toBe(false);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['failed', 'awaiting-health']);
|
||||
});
|
||||
|
||||
it('retains an interrupted startup slot while a newer destruction is still waiting', async () => {
|
||||
const jobs = await queueServers(2);
|
||||
const { processCreations, destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { interruptDeployments } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoCreateDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await processCreations();
|
||||
expect(destroyServer(jobs[0].name)).toMatchObject({ ok: true });
|
||||
db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ?").run(jobs[0].deploymentId);
|
||||
interruptDeployments();
|
||||
expect(await states()).toEqual(['interrupted', 'queued']);
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['awaiting-health', 'queued']);
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('releases a startup slot after its server is removed', async () => {
|
||||
const jobs = await queueServers(2);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await processCreations();
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(jobs[0].name);
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['failed', 'awaiting-health']);
|
||||
});
|
||||
|
||||
it('never verifies a replacement registered under the same server name', async () => {
|
||||
const jobs = await queueServers(2);
|
||||
const { processCreations } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await processCreations();
|
||||
db().prepare('UPDATE servers SET game_port = ? WHERE name = ?').run(28980, jobs[0].name);
|
||||
await processCreations();
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalledWith(jobs[0].name);
|
||||
expect(await states()).toEqual(['failed', 'awaiting-health']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('creation concurrency configuration', () => {
|
||||
it('defaults to one startup', async () => {
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', undefined);
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
expect(config.createConcurrency).toBe(1);
|
||||
});
|
||||
|
||||
it.each(['0', '-1', '1.5', 'invalid', 'Infinity', '9007199254740992'])('rejects invalid limit %s', async value => {
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', value);
|
||||
await expect(import('../../src/lib/server/config.js')).rejects.toThrow(/DEPLOYMENT_CREATE_CONCURRENCY/);
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1');
|
||||
});
|
||||
});
|
||||
@@ -36,6 +36,7 @@ beforeEach(async () => {
|
||||
vi.stubEnv('RCON_PASSWORD', 'private-rcon-password');
|
||||
vi.stubEnv('SERVER_PORT_MIN', '28960');
|
||||
vi.stubEnv('SERVER_PORT_MAX', '28980');
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1');
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
|
||||
|
||||
@@ -60,6 +61,7 @@ describe('multi-server blueprints', () => {
|
||||
const stored = db().prepare('SELECT recipe FROM server_blueprints WHERE id = ?').get(blueprint.id) as { recipe: string };
|
||||
expect(JSON.parse(stored.recipe)).toEqual({ format: 3, entries });
|
||||
expect(preview.ready).toBe(true);
|
||||
expect(preview.launchConcurrency).toBe(1);
|
||||
expect(preview.servers.map(server => server.name)).toEqual(['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1', 't6_zombie_town_1', 't6_zombie_town_2']);
|
||||
expect(preview.servers.map(server => server.port)).toEqual([28960, 28961, 28962, 28963, 28964]);
|
||||
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]);
|
||||
@@ -112,7 +114,7 @@ describe('multi-server blueprints', () => {
|
||||
expect(preview.entries[0].recipe.files).toContainEqual(expect.objectContaining({ path: 'telemetry.gsc', versionId: current.id }));
|
||||
expect(preview.entries[2].recipe.mapSlug).toBe('town');
|
||||
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
await processCreations();
|
||||
for (let pass = 0; pass <= preview.servers.length; pass++) await processCreations();
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, done: 5 });
|
||||
for (const name of ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1']) {
|
||||
expect(await readFile(join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/telemetry.gsc'), 'utf8')).toBe('current telemetry');
|
||||
@@ -221,9 +223,14 @@ describe('atomic collection launches', () => {
|
||||
const { komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
expect(komodoCreateDeployment).not.toHaveBeenCalled();
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, active: 5, done: 0, failed: 0 });
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ total: 5, active: 5, done: 0, failed: 0, queued: 5, provisioning: 0, verifying: 0 });
|
||||
expect(getCollection(queued.collectionId!)?.servers.map(server => server.queuePosition)).toEqual([1, 2, 3, 4, 5]);
|
||||
closeDb();
|
||||
await processCreations();
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ queued: 4, provisioning: 0, verifying: 1, done: 0 });
|
||||
expect(getCollection(queued.collectionId!)?.queue).toMatchObject({ concurrency: 1, queued: 4, active: [{ server: preview.servers[0].name, state: 'awaiting-health' }] });
|
||||
expect(getCollection(queued.collectionId!)?.servers.map(server => server.queuePosition)).toEqual([null, 1, 2, 3, 4]);
|
||||
for (let pass = 0; pass < preview.servers.length; pass++) await processCreations();
|
||||
const detail = getCollection(queued.collectionId!)!;
|
||||
expect(detail.collection).toMatchObject({ total: 5, active: 0, done: 5, failed: 0 });
|
||||
expect(listCollections()).toHaveLength(1);
|
||||
@@ -308,7 +315,7 @@ describe('atomic collection launches', () => {
|
||||
const newest = uploadNewVersion(script.id, Buffer.from('promoted while queued'));
|
||||
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(newest.id, script.id);
|
||||
uploadNewVersion(script.id, Buffer.from('unpromoted draft'));
|
||||
await processCreations();
|
||||
for (let pass = 0; pass <= preview.servers.length; pass++) await processCreations();
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ name: 'Friday Zombies', total: 5, done: 5 });
|
||||
for (const name of ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_moon_1']) {
|
||||
expect(await readFile(join(fixture.t5, name, 'runtime/plutonium/storage/t5/raw/scripts/sp/perks.gsc'), 'utf8')).toBe('promoted while queued');
|
||||
@@ -323,6 +330,9 @@ describe('atomic collection launches', () => {
|
||||
vi.mocked(komodoCreateDeployment).mockResolvedValueOnce({ ok: false, kind: 'http-error', message: 'Host rejected this container' });
|
||||
const queued = queueCollection({ previewId: preview.id, keys: keysFor(preview) });
|
||||
await processCreations();
|
||||
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ queued: 4, failed: 1, done: 0 });
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
for (let pass = 0; pass < preview.servers.length; pass++) await processCreations();
|
||||
const detail = getCollection(queued.collectionId!)!;
|
||||
expect(detail.collection).toMatchObject({ total: 5, active: 0, done: 4, failed: 1 });
|
||||
expect(detail.servers[0]).toMatchObject({ state: 'failed', error: expect.stringContaining('Host rejected') });
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'test-key');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'test-secret');
|
||||
});
|
||||
afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); });
|
||||
|
||||
describe('Komodo startup response classification', () => {
|
||||
it.each([403, 502])('preserves HTTP %i so callers can distinguish rejection from uncertain startup', async httpStatus => {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response('Deployment response failed', { status: httpStatus })));
|
||||
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(await komodoDeploy('t5_zombie_theater_1')).toMatchObject({ ok: false, kind: 'http-error', httpStatus });
|
||||
});
|
||||
|
||||
it('identifies an explicit success=false response as a confirmed HTTP 200 failure', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ success: false, message: 'Image rejected' })));
|
||||
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(await komodoDeploy('t5_zombie_theater_1')).toMatchObject({ ok: false, kind: 'http-error', httpStatus: 200, message: 'Image rejected' });
|
||||
});
|
||||
|
||||
it('returns a structured unreachable result when the startup response times out', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => { throw new DOMException('Request aborted', 'AbortError'); }));
|
||||
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(await komodoDeploy('t5_zombie_theater_1')).toMatchObject({ ok: false, kind: 'unreachable', message: expect.stringContaining('timed out') });
|
||||
});
|
||||
});
|
||||
@@ -100,6 +100,8 @@ describe('launch recipes and previews', () => {
|
||||
const recordedConfig = recipeForDeployment(queued.deploymentId)?.recipe.files.find(file => file.path === 'dedicated_sp.cfg');
|
||||
expect(Buffer.from(recordedConfig!.content, 'base64').toString()).toBe(deployedConfig);
|
||||
expect(getDeployment(queued.deploymentId)?.recipe?.fingerprint).not.toBe(preview.recipe.fingerprint);
|
||||
expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' });
|
||||
await processCreations();
|
||||
expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'done' });
|
||||
expect(JSON.stringify(getDeployment(queued.deploymentId))).not.toContain('private-');
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user