josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add configurable server destruction concurrency

Browse Source
This commit is contained in:
josh
2026-10-01 11:29:55 -04:00
parent 30030599d6
commit aa0223c35d
7 changed files with 359 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/destruction-concurrency.test.ts
+215
View File
@@ -0,0 +1,215 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
komodoCreateDeployment: vi.fn(),
komodoDeploy: vi.fn(),
komodoDestroyDeployment: vi.fn(),
komodoDeleteDeployment: vi.fn(),
komodoInspectHealthy: vi.fn(),
komodoEnsureTag: vi.fn(),
komodoUpdateTagsOnResource: vi.fn()
}));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
}));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
function deferred() {
let resolve!: () => void;
let reject!: (error: Error) => void;
const promise = new Promise<void>((done, fail) => { resolve = done; reject = fail; });
return { promise, resolve, reject };
}
function command() {
return { started: deferred(), response: deferred() };
}
const servers = Array.from({ length: 4 }, (_, index) => `t5_zombie_theater_${index + 1}`);
let fixture: Fixture;
let commands: Map<string, { destroy: ReturnType<typeof command>; delete: ReturnType<typeof command> }>;
let passes: Promise<void>[];
let runningAtSubmission: number[];
beforeEach(async () => {
vi.resetModules(); vi.resetAllMocks();
fixture = await newFixture(servers);
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test',
KOMODO_API_SECRET: 'test', KOMODO_TARGET_SERVER: 'test-host', RCON_PASSWORD: 'test' })) vi.stubEnv(key, value);
commands = new Map(servers.map(server => [server, { destroy: command(), delete: command() }]));
passes = [];
runningAtSubmission = [];
const { db } = await import('../../src/lib/server/db/index.js');
for (const [index, server] of servers.entries()) {
db().prepare(`INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, game_port, host_dir)
VALUES (?, 'deployment', ?, 't5', 'zombie_theater', 'zombie', 'theater', ?, ?)`)
.run(server, server, 28960 + index, join(fixture.t5, server));
}
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoDestroyDeployment, komodoDeleteDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
vi.mocked(komodoDestroyDeployment).mockImplementation(async server => {
runningAtSubmission.push((db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action = 'destroy' AND state = 'running'")
.get() as { n: number }).n);
const gate = commands.get(server)!.destroy;
gate.started.resolve();
await gate.response.promise;
return { ok: true };
});
vi.mocked(komodoDeleteDeployment).mockImplementation(async server => {
const gate = commands.get(server)!.delete;
gate.started.resolve();
await gate.response.promise;
return { ok: true };
});
});
afterEach(async () => {
for (const server of servers) complete(server);
await Promise.allSettled(passes);
await fixture.cleanup();
vi.unstubAllEnvs();
vi.restoreAllMocks();
});
function complete(server: string) {
commands.get(server)!.destroy.response.resolve();
commands.get(server)!.delete.response.resolve();
}
function track(pass: Promise<void>) {
passes.push(pass);
return pass;
}
async function queue(count = servers.length) {
const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
return servers.slice(0, count).map(server => {
const result = destroyServer(server);
if (!result.ok) throw new Error(result.message);
return result.deploymentId!;
});
}
async function states() {
const { db } = await import('../../src/lib/server/db/index.js');
return (db().prepare("SELECT state FROM server_deployments WHERE action = 'destroy' ORDER BY rowid")
.all() as { state: string }[]).map(row => row.state);
}
describe('destruction concurrency', () => {
it('defaults to one full removal at a time and refills the slot in the same pass', async () => {
await queue(2);
const { fleetSettings } = await import('../../src/lib/server/settings.js');
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
expect(fleetSettings().destroyConcurrency).toBe(1);
const pass = track(processDestructions());
await commands.get(servers[0])!.destroy.started.promise;
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
expect(await states()).toEqual(['running', 'waiting']);
complete(servers[0]);
await commands.get(servers[1])!.destroy.started.promise;
expect(await states()).toEqual(['done', 'running']);
complete(servers[1]);
await pass;
expect(await states()).toEqual(['done', 'done']);
expect(runningAtSubmission).toEqual([1, 1]);
});
it('runs two removals concurrently, holds slots through deletion, and coalesces overlapping passes', async () => {
await queue();
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
const pass = track(processDestructions());
await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise));
const overlapping = track(processDestructions());
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2);
expect(await states()).toEqual(['running', 'running', 'waiting', 'waiting']);
commands.get(servers[0])!.destroy.response.resolve();
await commands.get(servers[0])!.delete.started.promise;
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2);
commands.get(servers[0])!.delete.response.resolve();
await commands.get(servers[2])!.destroy.started.promise;
expect(await states()).toEqual(['done', 'running', 'running', 'waiting']);
complete(servers[1]);
await commands.get(servers[3])!.destroy.started.promise;
complete(servers[2]); complete(servers[3]);
await Promise.all([pass, overlapping]);
expect(await states()).toEqual(['done', 'done', 'done', 'done']);
expect(vi.mocked(komodoDestroyDeployment).mock.calls.map(([server]) => server)).toEqual(servers);
expect(komodoDeleteDeployment).toHaveBeenCalledTimes(4);
expect(Math.max(...runningAtSubmission)).toBe(2);
});
it('lets active removals finish after lowering the limit and applies it to new admissions', async () => {
await queue();
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
const pass = track(processDestructions());
await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise));
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 1 });
complete(servers[0]);
await vi.waitFor(async () => expect(await states()).toEqual(['done', 'running', 'waiting', 'waiting']));
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(2);
complete(servers[1]);
await commands.get(servers[2])!.destroy.started.promise;
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(3);
expect(await states()).toEqual(['done', 'done', 'running', 'waiting']);
complete(servers[2]);
await commands.get(servers[3])!.destroy.started.promise;
complete(servers[3]);
await pass;
expect(runningAtSubmission.slice(2)).toEqual([1, 1]);
});
it.each(['rejected', 'thrown'] as const)('releases capacity after a %s teardown failure', async failure => {
await queue(3);
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
if (failure === 'rejected') {
vi.mocked(komodoDestroyDeployment).mockImplementationOnce(async () => {
commands.get(servers[0])!.destroy.started.resolve();
await commands.get(servers[0])!.destroy.response.promise;
return { ok: false, kind: 'http-error', message: 'Teardown rejected' };
});
}
const pass = track(processDestructions());
await Promise.all(servers.slice(0, 2).map(server => commands.get(server)!.destroy.started.promise));
if (failure === 'thrown') commands.get(servers[0])!.destroy.response.reject(new Error('Connection lost'));
else commands.get(servers[0])!.destroy.response.resolve();
await commands.get(servers[2])!.destroy.started.promise;
expect(await states()).toEqual(['failed', 'running', 'running']);
complete(servers[1]); complete(servers[2]);
await pass;
expect(await states()).toEqual(['failed', 'done', 'done']);
expect(komodoDestroyDeployment).toHaveBeenCalledTimes(3);
expect(komodoDeleteDeployment).not.toHaveBeenCalledWith(servers[0]);
});
it('skips occupied and unknown servers while allowing later empty and stopped servers', async () => {
await queue();
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), destroyConcurrency: 2 });
const { processDestructions } = await import('../../src/lib/server/fleet/create.js');
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoDestroyDeployment, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
vi.mocked(rconGetPlayerCount).mockImplementation(async server => server === servers[0] ? 2 : server === servers[2] ? 0 : null);
vi.mocked(komodoInspectHealthy).mockImplementation(async server => server === servers[1]
? { ok: false, reason: 'Host unavailable' } : { ok: false, state: 'exited', reason: 'stopped' });
for (const server of servers) complete(server);
await track(processDestructions());
expect(await states()).toEqual(['waiting', 'waiting', 'done', 'done']);
expect(vi.mocked(komodoDestroyDeployment).mock.calls.map(([server]) => server)).toEqual(servers.slice(2));
expect(rconGetPlayerCount).toHaveBeenCalledTimes(4);
});
});
tests/unit/fleet-settings.test.ts
+21 -4
View File
@@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const defaults = { deployKeepDays: 7, restartConcurrency: 1, createConcurrency: 1 };
const defaults = { deployKeepDays: 7, restartConcurrency: 1, createConcurrency: 1, destroyConcurrency: 1 };
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
@@ -14,7 +14,7 @@ describe('fleet settings API and persistence', () => {
it('returns defaults and saves all settings across a module reload and database reopen', async () => {
const { GET, PUT } = await import('../../src/routes/api/settings/+server.js');
expect(await GET().json()).toEqual(defaults);
const saved = { deployKeepDays: 30, restartConcurrency: 3, createConcurrency: 2 };
const saved = { deployKeepDays: 30, restartConcurrency: 3, createConcurrency: 2, destroyConcurrency: 4 };
const response = await PUT({ request: new Request('http://localhost/api/settings', { method: 'PUT', body: JSON.stringify(saved) }) });
expect(response.status).toBe(200);
expect(await response.json()).toEqual(saved);
@@ -37,17 +37,34 @@ describe('fleet settings API and persistence', () => {
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '4');
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '2');
const { fleetSettings } = await import('../../src/lib/server/settings.js');
expect(fleetSettings()).toEqual({ deployKeepDays: 14, restartConcurrency: 4, createConcurrency: 2 });
expect(fleetSettings()).toEqual({ deployKeepDays: 14, restartConcurrency: 4, createConcurrency: 2, destroyConcurrency: 1 });
closeDb();
vi.stubEnv('DEPLOY_KEEP_DAYS', '7');
expect(fleetSettings().deployKeepDays).toBe(14);
expect(db().prepare("SELECT value FROM settings WHERE key = 'unrelated'").get()).toEqual({ value: 'true' });
});
it('upgrades saved settings with a default destruction limit while retaining existing values', async () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const previous = { deployKeepDays: 21, restartConcurrency: 3, createConcurrency: 2 };
db().prepare("UPDATE settings SET value = ? WHERE key = 'fleet_settings'").run(JSON.stringify(previous));
closeDb();
vi.stubEnv('DEPLOY_KEEP_DAYS', 'bad');
vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '9');
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '8');
const { GET } = await import('../../src/routes/api/settings/+server.js');
const upgraded = { ...previous, destroyConcurrency: 1 };
expect(await GET().json()).toEqual(upgraded);
expect(JSON.parse((db().prepare("SELECT value FROM settings WHERE key = 'fleet_settings'").get() as { value: string }).value)).toEqual(upgraded);
closeDb();
expect(await GET().json()).toEqual(upgraded);
});
it.each([
'null', '[]', '{}', '{',
...['deployKeepDays', 'restartConcurrency', 'createConcurrency'].flatMap(key =>
...['deployKeepDays', 'restartConcurrency', 'createConcurrency', 'destroyConcurrency'].flatMap(key =>
[0, -1, 1.5, '2', true, null, 9007199254740992].map(value => JSON.stringify({ ...defaults, [key]: value }))),
JSON.stringify({ deployKeepDays: 7, restartConcurrency: 1, createConcurrency: 1 }),
JSON.stringify({ ...defaults, deployKeepDays: 36501 }),
JSON.stringify({ ...defaults, unknown: 1 })
])('rejects invalid input without partially saving: %s', async body => {