Add configurable server destruction concurrency
This commit is contained in:
@@ -0,0 +1,215 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoCreateDeployment: vi.fn(),
|
||||
komodoDeploy: vi.fn(),
|
||||
komodoDestroyDeployment: vi.fn(),
|
||||
komodoDeleteDeployment: vi.fn(),
|
||||
komodoInspectHealthy: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(),
|
||||
komodoUpdateTagsOnResource: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
function deferred() {
|
||||
let resolve!: () => void;
|
||||
let reject!: (error: Error) => void;
|
||||
const promise = new Promise<void>((done, fail) => { resolve = done; reject = fail; });
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
function command() {
|
||||
return { started: deferred(), response: deferred() };
|
||||
}
|
||||
|
||||
const servers = Array.from({ length: 4 }, (_, index) => `t5_zombie_theater_${index + 1}`);
|
||||
let fixture: Fixture;
|
||||
let commands: Map<string, { destroy: ReturnType<typeof command>; delete: ReturnType<typeof command> }>;
|
||||
let passes: Promise<void>[];
|
||||
let runningAtSubmission: number[];
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.resetAllMocks();
|
||||
fixture = await newFixture(servers);
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test',
|
||||
KOMODO_API_SECRET: 'test', KOMODO_TARGET_SERVER: 'test-host', RCON_PASSWORD: 'test' })) vi.stubEnv(key, value);
|
||||
commands = new Map(servers.map(server => [server, { destroy: command(), delete: command() }]));
|
||||
passes = [];
|
||||
runningAtSubmission = [];
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
for (const [index, server] of servers.entries()) {
|
||||
db().prepare(`INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, game_port, host_dir)
|
||||
VALUES (?, 'deployment', ?, 't5', 'zombie_theater', 'zombie', 'theater', ?, ?)`)
|
||||
.run(server, server, 28960 + index, join(fixture.t5, server));
|
||||
}
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
vi.mocked(komodoDestroyDeployment).mockImplementation(async server => {
|
||||
runningAtSubmission.push((db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action = 'destroy' AND state = 'running'")
|
||||
.get() as { n: number }).n);
|
||||
const gate = commands.get(server)!.destroy;
|
||||
gate.started.resolve();
|
||||
await gate.response.promise;
|
||||
return { ok: true };
|
||||
});
|
||||
vi.mocked(komodoDeleteDeployment).mockImplementation(async server => {
|
||||
const gate = commands.get(server)!.delete;
|
||||
gate.started.resolve();
|
||||
await gate.response.promise;
|
||||
return { ok: true };
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
for (const server of servers) complete(server);
|
||||
await Promise.allSettled(passes);
|
||||
await fixture.cleanup();
|
||||
vi.unstubAllEnvs();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function complete(server: string) {
|
||||
commands.get(server)!.destroy.response.resolve();
|
||||
commands.get(server)!.delete.response.resolve();
|
||||
}
|
||||
|
||||
function track(pass: Promise<void>) {
|
||||
passes.push(pass);
|
||||
return pass;
|
||||
}
|
||||
|
||||
async function queue(count = servers.length) {
|
||||
const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
return servers.slice(0, count).map(server => {
|
||||
const result = destroyServer(server);
|
||||
if (!result.ok) throw new Error(result.message);
|
||||
return result.deploymentId!;
|
||||
});
|
||||
}
|
||||
|
||||
async function states() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
return (db().prepare("SELECT state FROM server_deployments WHERE action = 'destroy' ORDER BY rowid")
|
||||
.all() as { state: string }[]).map(row => row.state);
|
||||
}
|
||||
|
||||
describe('destruction concurrency', () => {
|
||||
it('defaults to one full removal at a time and refills the slot in the same pass', async () => {
|
||||
await queue(2);
|
||||
const { fleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(fleetSettings().destroyConcurrency).toBe(1);
|
||||
const pass = track(processDestructions());
|
||||
await commands.get(servers[0])!.destroy.started.promise;
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(await states()).toEqual(['running', 'waiting']);
|
||||
complete(servers[0]);
|
||||
await commands.get(servers[1])!.destroy.started.promise;
|
||||
expect(await states()).toEqual(['done', 'running']);
|
||||
complete(servers[1]);
|
||||
await pass;
|
||||
expect(await states()).toEqual(['done', 'done']);
|
||||
expect(runningAtSubmission).toEqual([1, 1]);
|
||||
});
|
||||
|
||||
it('runs two removals concurrently, holds slots through deletion, and coalesces overlapping passes', async () => {
|
||||
await queue();
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
|
||||
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const pass = track(processDestructions());
|
||||
await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise));
|
||||
const overlapping = track(processDestructions());
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(await states()).toEqual(['running', 'running', 'waiting', 'waiting']);
|
||||
commands.get(servers[0])!.destroy.response.resolve();
|
||||
await commands.get(servers[0])!.delete.started.promise;
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2);
|
||||
commands.get(servers[0])!.delete.response.resolve();
|
||||
await commands.get(servers[2])!.destroy.started.promise;
|
||||
expect(await states()).toEqual(['done', 'running', 'running', 'waiting']);
|
||||
complete(servers[1]);
|
||||
await commands.get(servers[3])!.destroy.started.promise;
|
||||
complete(servers[2]); complete(servers[3]);
|
||||
await Promise.all([pass, overlapping]);
|
||||
expect(await states()).toEqual(['done', 'done', 'done', 'done']);
|
||||
expect(vi.mocked(komodoDestroyDeployment).mock.calls.map(([server]) => server)).toEqual(servers);
|
||||
expect(komodoDeleteDeployment).toHaveBeenCalledTimes(4);
|
||||
expect(Math.max(...runningAtSubmission)).toBe(2);
|
||||
});
|
||||
|
||||
it('lets active removals finish after lowering the limit and applies it to new admissions', async () => {
|
||||
await queue();
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
|
||||
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const pass = track(processDestructions());
|
||||
await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise));
|
||||
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 1 });
|
||||
complete(servers[0]);
|
||||
await vi.waitFor(async () => expect(await states()).toEqual(['done', 'running', 'waiting', 'waiting']));
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2);
|
||||
complete(servers[1]);
|
||||
await commands.get(servers[2])!.destroy.started.promise;
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(3);
|
||||
expect(await states()).toEqual(['done', 'done', 'running', 'waiting']);
|
||||
complete(servers[2]);
|
||||
await commands.get(servers[3])!.destroy.started.promise;
|
||||
complete(servers[3]);
|
||||
await pass;
|
||||
expect(runningAtSubmission.slice(2)).toEqual([1, 1]);
|
||||
});
|
||||
|
||||
it.each(['rejected', 'thrown'] as const)('releases capacity after a %s teardown failure', async failure => {
|
||||
await queue(3);
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
|
||||
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
if (failure === 'rejected') {
|
||||
vi.mocked(komodoDestroyDeployment).mockImplementationOnce(async () => {
|
||||
commands.get(servers[0])!.destroy.started.resolve();
|
||||
await commands.get(servers[0])!.destroy.response.promise;
|
||||
return { ok: false, kind: 'http-error', message: 'Teardown rejected' };
|
||||
});
|
||||
}
|
||||
const pass = track(processDestructions());
|
||||
await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise));
|
||||
if (failure === 'thrown') commands.get(servers[0])!.destroy.response.reject(new Error('Connection lost'));
|
||||
else commands.get(servers[0])!.destroy.response.resolve();
|
||||
await commands.get(servers[2])!.destroy.started.promise;
|
||||
expect(await states()).toEqual(['failed', 'running', 'running']);
|
||||
complete(servers[1]); complete(servers[2]);
|
||||
await pass;
|
||||
expect(await states()).toEqual(['failed', 'done', 'done']);
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(3);
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalledWith(servers[0]);
|
||||
});
|
||||
|
||||
it('skips occupied and unknown servers while allowing later empty and stopped servers', async () => {
|
||||
await queue();
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
|
||||
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(rconGetPlayerCount).mockImplementation(async server => server === servers[0] ? 2 : server === servers[2] ? 0 : null);
|
||||
vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === servers[1]
|
||||
? { ok: false, reason: 'Host unavailable' } : { ok: false, state: 'exited', reason: 'stopped' });
|
||||
for (const server of servers) complete(server);
|
||||
await track(processDestructions());
|
||||
expect(await states()).toEqual(['waiting', 'waiting', 'done', 'done']);
|
||||
expect(vi.mocked(komodoDestroyDeployment).mock.calls.map(([server]) => server)).toEqual(servers.slice(2));
|
||||
expect(rconGetPlayerCount).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user