josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Queue native collection launches before async placement

Browse Source
This commit is contained in:
josh
2026-10-05 11:35:50 -04:00
parent 1405114a14
commit 7c76851252
34 changed files with 1324 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/collection-match-performance.test.ts
+105 -8
View File
@@ -12,10 +12,14 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
let fixture: Fixture;
let hostId: string;
let agentCredential: string;
let heartbeatInventory: Record<string, unknown>;
let releasePlacement: (() => void) | undefined;
const image = 'example.test/plutainer:test';
beforeEach(async () => {
vi.resetModules(); vi.clearAllMocks();
releasePlacement = undefined;
fixture = await newFixture([]);
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
vi.stubEnv('PLUTAINER_IMAGE', image);
@@ -31,7 +35,7 @@ beforeEach(async () => {
protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
storageAvailable: 1_000_000_000_000, runtimeImages: [image]
};
enrollHost(createEnrollment(hostId).token, inventory);
agentCredential = enrollHost(createEnrollment(hostId).token, inventory).credential;
const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
const { db } = await import('../../src/lib/server/db/index.js');
const libraries = ['t5', 't6'].map(game => {
@@ -41,7 +45,8 @@ beforeEach(async () => {
return { id, state: 'ready' as const };
});
updateHostLibraryCaches(hostId, libraries.map(library => ({ libraryVersionId: library.id, status: 'ready', manifestSha256: 'a'.repeat(64) })));
heartbeat(hostId, { ...inventory, libraries });
heartbeatInventory = { ...inventory, libraries };
heartbeat(hostId, heartbeatInventory);
const { T6_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
const now = new Date().toISOString();
db().transaction(() => {
@@ -68,6 +73,7 @@ beforeEach(async () => {
});
afterEach(async () => {
releasePlacement?.();
await new Promise<void>(resolve => setImmediate(resolve));
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
@@ -78,7 +84,44 @@ afterEach(async () => {
});
describe('matching a native collection', () => {
it('admits nine removals and ten creations without copying plugin bytes into every queued launch', async () => {
it('places queued creations and completes all exact removals during later worker passes', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 10, destroyConcurrency: 9 });
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const { T5_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
const blueprint = saveBlueprint({ name: 'Replace nine with ten', entries: T5_ZOMBIE_MAPS.filter(map => map.slug !== 'zombietron')
.map(map => ({ game: 't5', mapSlug: map.slug, count: 1 })) });
const preview = previewCollection(blueprint.id, 'exact');
const queued = queueCollection({ previewId: preview.id,
keys: Object.fromEntries(preview.servers.map(server => [server.name!, `key-${server.name}`])) });
expect(queueCollection({ previewId: preview.id })).toEqual(queued);
expect(db().prepare('SELECT id FROM native_instances').all()).toHaveLength(9);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(18);
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
const operations = db().prepare("SELECT id,kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.remove')")
.all() as { id: string; kind: string }[];
expect(operations.filter(operation => operation.kind === 'instance.create')).toHaveLength(10);
const removals = operations.filter(operation => operation.kind === 'instance.remove');
expect(removals).toHaveLength(9);
const { reportHostOperationStatus } = await import('../../src/lib/server/hosts/index.js');
for (const operation of removals) reportHostOperationStatus(hostId, operation.id, { state: 'completed', result: { exists: false } });
await processProvisioning();
const active = db().prepare('SELECT id,port FROM native_instances WHERE retired_at IS NULL').all() as { id: string; port: number }[];
expect(active).toHaveLength(10);
expect(new Set(active.map(instance => instance.port)).size).toBe(10);
for (const instance of active) expect(db().prepare('SELECT protocol,port FROM ec_host_ports WHERE instance_id=? ORDER BY protocol').all(instance.id))
.toEqual([{ protocol: 'tcp', port: instance.port }, { protocol: 'udp', port: instance.port }]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(20);
expect(getCollection(queued.collectionId!)?.servers.filter(server => server.action === 'destroy').map(server => server.state))
.toEqual(Array(9).fill('done'));
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.remove')").all()).toHaveLength(19);
});
it('accepts nine removals and ten unplaced creations before capacity checks or file loading', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { createScript, setScope, deleteScript } = await import('../../src/lib/server/fleet/scripts.js');
const content = Buffer.alloc(8 * 1024 ** 2, 42);
@@ -91,6 +134,10 @@ describe('matching a native collection', () => {
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { POST } = await import('../../src/routes/api/deployment-collections/+server.js');
const { load } = await import('../../src/routes/collections/launches/[id]/+page.server.js');
const { GET: collectionResponse } = await import('../../src/routes/api/deployment-collections/[id]/+server.js');
const { GET: fleetResponse } = await import('../../src/routes/api/servers/+server.js');
const { POST: agentHeartbeat } = await import('../../src/routes/api/agent/heartbeat/+server.js');
const { T5_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
const blueprint = saveBlueprint({ name: 'Match ten T5 lobbies', entries: T5_ZOMBIE_MAPS.filter(map => map.slug !== 'zombietron')
.map(map => ({ game: 't5', mapSlug: map.slug, count: 1 })) });
@@ -102,6 +149,23 @@ describe('matching a native collection', () => {
.get(preview.id) as { bytes: number };
expect(storedPreview.bytes).toBeLessThan(100_000);
const capacity = await import('../../src/lib/server/hosts/capacity.js');
const placement = vi.spyOn(capacity, 'choosePlacement').mockImplementation(() => {
throw new Error('Placement is waiting for fresh host capacity');
});
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
const evaluate = preparation.evaluateNativePlacement;
let placementStarted!: () => void;
const started = new Promise<void>(resolve => { placementStarted = resolve; });
const gate = new Promise<void>(resolve => { releasePlacement = resolve; });
const evaluation = vi.spyOn(preparation, 'evaluateNativePlacement').mockImplementation(async options => {
placementStarted();
await gate;
return evaluate(options);
});
const scripts = await import('../../src/lib/server/fleet/scripts.js');
const fullVersion = vi.spyOn(scripts, 'getVersion');
const submit = (keys?: Record<string, string>) => POST({ request: new Request('http://localhost/api/deployment-collections', {
method: 'POST', body: JSON.stringify({ previewId: preview.id, keys }) }) });
const response = await submit(Object.fromEntries(preview.servers.map(server => [server.name!, `key-${server.name}`])));
@@ -109,21 +173,54 @@ describe('matching a native collection', () => {
expect(response.status).toBe(202);
expect(body).toMatchObject({ ok: true, collectionId: expect.any(String) });
expect(await (await submit()).json()).toEqual(body);
expect(load({ params: { id: body.collectionId } })).toEqual({});
// Exercise the real route and worker: admission must return before
// loading launch payloads or inspecting the nine servers to remove.
expect(nativeRequest).not.toHaveBeenCalled();
expect(placement).not.toHaveBeenCalled();
expect(evaluation).not.toHaveBeenCalled();
expect(fullVersion).not.toHaveBeenCalled();
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]);
for (const table of ['server_launch_previews', 'server_launch_recipes']) {
const rows = db().prepare(`SELECT length(recipe) AS bytes FROM ${table}`).all() as { bytes: number }[];
expect(rows).toHaveLength(10);
expect(rows.every(row => row.bytes < 10_000)).toBe(true);
}
expect(db().prepare('SELECT id FROM native_instances').all()).toHaveLength(9);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(18);
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toHaveLength(10);
expect(db().prepare('SELECT id FROM server_launch_previews').all()).toEqual([]);
const rows = db().prepare('SELECT length(recipe) AS bytes FROM server_launch_recipes').all() as { bytes: number }[];
expect(rows).toHaveLength(10);
expect(rows.every(row => row.bytes < 10_000)).toBe(true);
const collection = getCollection(body.collectionId)!;
expect(collection.collection).toMatchObject({ mode: 'exact', total: 19 });
expect(collection.servers.filter(server => server.action === 'destroy')).toHaveLength(9);
expect(collection.servers.filter(server => server.action === 'create')).toHaveLength(10);
const { nativeDeployment } = await import('../../src/lib/server/backends/native.js');
for (const server of collection.servers.filter(server => server.action === 'create')) {
expect(nativeDeployment(server.id)).toBe(true);
expect(getDeployment(server.id)?.backend).toBe('native');
}
await new Promise<void>(resolve => setImmediate(resolve));
await processProvisioning({ wait: false });
await started;
expect(evaluation).toHaveBeenCalled();
expect(placement).not.toHaveBeenCalled();
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]);
expect(getCollection(body.collectionId)?.collection.total).toBe(19);
const collectionPage = collectionResponse({ params: { id: body.collectionId } });
expect(collectionPage.status).toBe(200);
expect((await collectionPage.json()).servers).toHaveLength(19);
const fleet = fleetResponse();
expect(fleet.status).toBe(200);
expect((await fleet.json()).servers).toHaveLength(9);
const heartbeatResponse = await agentHeartbeat({ request: new Request('https://controller.test/api/agent/heartbeat', {
method: 'POST', headers: { authorization: `Bearer ${agentCredential}` }, body: JSON.stringify({ inventory: heartbeatInventory })
}) });
expect(heartbeatResponse.status).toBe(200);
expect(await heartbeatResponse.json()).toMatchObject({ host: { id: hostId, connected: true }, heartbeatSeconds: 15 });
// All three endpoints complete while the background placement is still paused.
expect(placement).not.toHaveBeenCalled();
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]);
placement.mockRestore(); fullVersion.mockRestore();
releasePlacement!();
await processProvisioning();
const dispatched = db().prepare(`SELECT n.deployment_id,o.id FROM native_provisioning n
JOIN ec_host_operations o ON o.id=n.operation_id WHERE o.kind='instance.create'`).all() as { deployment_id: string; id: string }[];