josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Queue native collection launches before async placement

Browse Source
This commit is contained in:
josh
2026-10-05 11:35:50 -04:00
parent 1405114a14
commit 7c76851252
34 changed files with 1324 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/native-provisioning.test.ts
+141 -34
View File
@@ -43,10 +43,11 @@ async function prepareLibrary(game: 't5' | 't6' = 't5') {
const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory());
return ready;
}
async function queue(game: 't5' | 't6' = 't5') {
async function queue(game: 't5' | 't6' = 't5', placement: 'pinned' | 'automatic' = 'pinned') {
await prepareLibrary(game); remote.map = game === 't5' ? 'zombie_theater' : 'zm_transit';
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
const result = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town', hostId, libraryId, image, plutoKey: 'saved-key' });
const result = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town',
hostId: placement === 'pinned' ? hostId : undefined, libraryId, image, plutoKey: 'saved-key' });
if (!result.ok) throw new Error(result.message);
return result;
}
@@ -73,6 +74,44 @@ async function detail(id: string) {
}
describe('native instance provisioning', () => {
it('recognizes an unplaced native intent and resumes it once after a controller restart', async () => {
const job = await queue();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const { nativeDeployment, nativeInstance } = await import('../../src/lib/server/backends/native.js');
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
expect(job.port).toBeNull();
expect(nativeDeployment(job.deploymentId)).toBe(true);
expect(nativeInstance(job.name)).toBeNull();
expect(getDeployment(job.deploymentId)).toMatchObject({ backend: 'native', canCancelCreation: true });
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([{ deployment_id: job.deploymentId }]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
closeDb(); vi.resetModules();
const backend = await import('../../src/lib/server/backends/native.js');
expect(backend.nativeDeployment(job.deploymentId)).toBe(true);
await advance();
const original = await operationFor(job.deploymentId);
expect(original).toMatchObject({ kind: 'instance.create', payload: { name: job.name, port: 28960 } });
await advance();
expect((await operationFor(job.deploymentId))?.id).toBe(original!.id);
const restartedDb = (await import('../../src/lib/server/db/index.js')).db();
expect(restartedDb.prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
expect(restartedDb.prepare('SELECT protocol FROM ec_host_ports ORDER BY protocol').all()).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]);
});
it('still rejects missing keys and stale lobby identities before creating an intent', async () => {
await prepareLibrary();
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', hostId, libraryId, image };
expect(queueNativeCreation(input)).toMatchObject({ ok: false, message: expect.stringContaining('server key') });
expect(queueNativeCreation({ ...input, plutoKey: 'saved-key', expectedName: 't5_zombie_theater_2' }))
.toMatchObject({ ok: false });
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([]);
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
});
it('accepts cancellation through the provisioning API and rejects an already completed creation', async () => {
const job = await queue();
const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js');
@@ -341,11 +380,30 @@ describe('native instance provisioning', () => {
expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ count: 0 });
});
it('keeps a queued server pinned to its accepted files when newer game files arrive', async () => {
const job = await queue();
it.each(['pinned', 'automatic'] as const)('keeps an unplaced %s launch pinned to its accepted files during cleanup', async placement => {
const job = await queue('t5', placement);
const acceptedId = libraryId;
const latest = await prepareLibrary('t5');
expect(latest.id).not.toBe(acceptedId);
const { db } = await import('../../src/lib/server/db/index.js');
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
const { reconcileHostLibraryCleanup, cleanupControllerLibraries, getLibraryVersion, updateHostLibraryCaches }
= await import('../../src/lib/server/game-files/index.js');
const accepted = getLibraryVersion(acceptedId)!;
heartbeat(hostId, { ...inventory(), capabilities: ['library.remove'] });
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
reconcileHostLibraryCleanup(hostId);
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=?")
.all(acceptedId)).toEqual([]);
// The unplaced intent also protects the controller's copy when every
// host reports it absent; its accepted version remains downloadable.
updateHostLibraryCaches(hostId, [{ libraryVersionId: acceptedId, status: 'missing' }]);
await cleanupControllerLibraries();
expect(db().prepare('SELECT version_id FROM game_file_cleanup WHERE version_id=?').get(acceptedId)).toBeUndefined();
const { stat } = await import('node:fs/promises');
expect((await stat(join(fixture.root, 'libraries', 't5', acceptedId, 'zone/base.ff'))).isFile()).toBe(true);
updateHostLibraryCaches(hostId, [{ libraryVersionId: acceptedId, status: 'ready', manifestSha256: accepted.manifestSha256 }]);
await advance();
expect((await operationFor(job.deploymentId))?.payload.libraryId).toBe(acceptedId);
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
@@ -355,7 +413,8 @@ describe('native instance provisioning', () => {
it.each(['t5', 't6'] as const)('queues and verifies a %s deployment without Komodo', async game => {
const job = await queue(game); await advance();
const operation = await operationFor(job.deploymentId);
expect(operation).toMatchObject({ kind: 'instance.create', hostId, generation: 1, state: 'pending', payload: { game, libraryId, image, port: job.port } });
expect(job.port).toBeNull();
expect(operation).toMatchObject({ kind: 'instance.create', hostId, generation: 1, state: 'pending', payload: { game, libraryId, image, port: 28960 } });
expect(operation?.payload.files).toEqual(expect.arrayContaining([expect.objectContaining({ path: 'configs/electric_cherry_map.cfg' })]));
await acknowledge(job.deploymentId);
expect((await detail(job.deploymentId)).state).toBe('done');
@@ -534,17 +593,22 @@ describe('native instance provisioning', () => {
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe(createHash('sha256').update(after).digest('hex'));
});
it('atomically launches mixed-game collections with per-host port reservations', async () => {
it('atomically queues mixed-game collections and reserves per-host ports when dispatched', async () => {
const t5 = await prepareLibrary('t5'); const t6 = await prepareLibrary('t6');
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const blueprint = saveBlueprint({ name: 'Both games', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }, { game: 't6', mapSlug: 'town', count: 1 }] });
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const preview = previewCollection(blueprint.id, 'fill', { hostId, libraryIds: { t5: t5.id, t6: t6.id }, image });
expect(preview.ready).toBe(true);
expect(preview.servers.map(server => server.port)).toEqual([28960, 28961, 28962]);
expect(preview.servers.map(server => server.port)).toEqual([null, null, null]);
const queued = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) });
expect(queueCollection({ previewId: preview.id }).collectionId).toBe(queued.collectionId);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 });
await advance();
const instances = db().prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port').all();
expect(instances).toEqual([{ host_id: hostId, library_id: t5.id, port: 28960 }, { host_id: hostId, library_id: t5.id, port: 28961 }, { host_id: hostId, library_id: t6.id, port: 28962 }]);
const { createHost, createEnrollment, enrollHost, hostPortAvailable } = await import('../../src/lib/server/hosts/index.js');
@@ -554,13 +618,15 @@ describe('native instance provisioning', () => {
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'moon', hostId: secondHost.id, libraryId: t5.id, image, plutoKey: 'second-host-key' });
expect(second.ok).toBe(true); if (!second.ok) throw new Error(second.message);
expect(second.port).toBe(28960);
expect(second.port).toBeNull();
await advance();
expect((await operationFor(second.deploymentId))?.payload.port).toBe(28960);
expect(hostPortAvailable(hostId, 28960)).toBe(false);
expect(hostPortAvailable(secondHost.id, 28960)).toBe(false);
expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ count: 8 });
});
it('reads remote instance files through the agent including missing files and epoch timestamps', async () => {
const job = await queue();
const job = await queue(); await advance();
const backend = await import('../../src/lib/server/backends/native.js');
const remoteRead = vi.mocked(backend.nativeRequest);
remoteRead.mockImplementation(async (_instance, kind, payload) => {
@@ -622,17 +688,17 @@ describe('native instance provisioning', () => {
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
expect(getServerScript(job.name, 'hud.gsc')).toBeUndefined();
});
it('cancels a queued native launch locally and releases both ports without submitting a create', async () => {
it('cancels an unplaced native launch locally without reserving ports or submitting a create', async () => {
const job = await queue();
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
const instance = nativeInstance(job.name)!;
expect(nativeInstance(job.name)).toBeNull();
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
const cancelled = destroyServer(job.name); expect(cancelled.ok).toBe(true);
await advance();
const { db } = await import('../../src/lib/server/db/index.js');
expect((await detail(job.deploymentId)).state).toBe('cancelled');
expect(nativeInstance(job.name)).toBeNull();
expect(db().prepare('SELECT * FROM ec_host_ports WHERE instance_id=?').all(instance.id)).toEqual([]);
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind LIKE 'instance.%'").get()).toEqual({ count: 0 });
const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js');
expect(getLibraryVersion(libraryId)?.state).toBe('ready');
@@ -661,7 +727,8 @@ describe('automatic capacity admission and placement', () => {
const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
const result = queueCreateServer(input);
expect(result.ok).toBe(true); if (!result.ok) return;
expect(nativeInstance(result.name)?.host_id).toBe(hostId); // Fast first fit.
expect(nativeInstance(result.name)).toBeNull();
expect(result.port).toBeNull();
await advance();
expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id);
expect(nativeInstance(result.name)).toMatchObject({ host_id: other.id, public_address: 'second.example' });
@@ -689,13 +756,18 @@ describe('automatic capacity admission and placement', () => {
it.each([
{ cpuUsagePercent: undefined }, { memoryAvailable: undefined }, { cpuCount: 0 },
{ cpuUsagePercent: 99 }, { memoryAvailable: 1024 ** 3 }
])('rejects admission with unavailable resources: %j', async overrides => {
])('accepts an unplaced job and waits for unavailable resources: %j', async overrides => {
await prepareLibrary();
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), ...overrides });
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const { db } = await import('../../src/lib/server/db/index.js');
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict', message: expect.stringContaining('capacity') });
const queued = queueNativeCreation(input);
expect(queued).toMatchObject({ ok: true, port: null });
if (!queued.ok) throw new Error(queued.message);
await advance();
expect(await operationFor(queued.deploymentId)).toBeNull();
expect((await detail(queued.deploymentId)).stage).toContain('capacity');
expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
});
@@ -709,22 +781,28 @@ describe('automatic capacity admission and placement', () => {
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId);
const result = queueNativeCreation({ ...input, previewId: preview.id });
expect(result).toMatchObject({ ok: true, name: preview.name, port: 30000 });
expect(result).toMatchObject({ ok: true, name: preview.name, port: null });
expect(queueNativeCreation({ ...input, previewId: preview.id })).toEqual(result);
if (!result.ok) throw new Error(result.message);
await advance();
expect((await operationFor(result.deploymentId))?.payload.port).toBe(30000);
});
it('rejects an entire collection when capacity shrinks after its preview', async () => {
it('queues the whole collection and waits when capacity shrinks after its preview', async () => {
await prepareLibrary();
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
const preview = previewCollection(blueprint.id);
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
expect(() => queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) })).toThrow('capacity');
heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 });
const result = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) });
expect(result).toMatchObject({ ok: true, collectionId: expect.any(String) });
await advance();
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toHaveLength(1);
expect(db().prepare('SELECT * FROM native_creation_intents').all()).toHaveLength(2);
});
it.each([
{ resource: 'CPU', measurements: { cpuCount: 4, cpuUsagePercent: 82.5 }, over: { cpuUsagePercent: 82.51 } },
@@ -742,15 +820,19 @@ describe('automatic capacity admission and placement', () => {
expect(() => choosePlacement(options)).toThrow('capacity');
expect(() => choosePlacement({ ...options, thorough: true })).toThrow('capacity');
});
it('uses calibrated footprints to admit more instances than the cold-start allowance', async () => {
it('dispatches a waiting instance after calibration lowers the cold-start allowance', async () => {
const first = await queue();
await advance(); await acknowledge(first.deploymentId);
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
const { recordInstanceResources, capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const measured = { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 };
const measured = { ...inventory(), cpuCount: 1, cpuUsagePercent: 50 };
heartbeat(hostId, measured);
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
const next = queueNativeCreation(input);
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
await advance();
expect(await operationFor(next.deploymentId)).toBeNull();
const instance = nativeInstance(first.name)!;
for (let minute = 0; minute < 25; minute++) {
recordInstanceResources(hostId, { ...measured, instanceResources: [{ instanceId: instance.id, generation: 1,
@@ -758,7 +840,8 @@ describe('automatic capacity admission and placement', () => {
new Date(Date.now() - (25 - minute) * 60_000).toISOString());
}
expect(capacityCalibration(hostId).startup.source).toBe('host');
expect(queueNativeCreation(input)).toMatchObject({ ok: true });
await advance();
expect(await operationFor(next.deploymentId)).not.toBeNull();
});
it('queues a collection at running demand and releases startup reservations as live usage arrives', async () => {
await prepareLibrary();
@@ -791,7 +874,7 @@ describe('automatic capacity admission and placement', () => {
await advance();
expect(creates()).toHaveLength(2); // Its actual usage replaces the peak even before health verification.
});
it('charges queued reservations and releases them on cancellation', async () => {
it('reserves queued lobby identities without host resources and releases them on cancellation', async () => {
await prepareLibrary();
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
@@ -799,9 +882,13 @@ describe('automatic capacity admission and placement', () => {
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
const first = queueNativeCreation(input);
expect(first.ok).toBe(true); if (!first.ok) return;
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
const second = queueNativeCreation(input);
expect(second).toMatchObject({ ok: true, name: 't5_zombie_theater_2', port: null });
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
cancelCreation(first.deploymentId);
expect(queueNativeCreation(input)).toMatchObject({ ok: true });
expect(queueNativeCreation(input)).toMatchObject({ ok: true, name: first.name, port: null });
});
it.each(['memory', 'cpu'] as const)('dispatches immediately after %s recovers instead of waiting for five-minute history', async resource => {
await prepareLibrary();
@@ -846,7 +933,10 @@ describe('automatic capacity admission and placement', () => {
// even though the next host heartbeat has not observed it yet.
await advance();
expect(db().prepare("SELECT * FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
const later = queueNativeCreation(input);
expect(later.ok).toBe(true); if (!later.ok) throw new Error(later.message);
await advance();
expect(await operationFor(later.deploymentId)).toBeNull();
});
it('does not charge a starting container twice when its usage is already in the heartbeat', async () => {
await prepareLibrary();
@@ -873,6 +963,8 @@ describe('automatic capacity admission and placement', () => {
});
it.each(['missing', 'old-generation'] as const)('keeps full reservations for a submitted launch with %s measurements', async evidence => {
await prepareLibrary();
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
@@ -885,8 +977,11 @@ describe('automatic capacity admission and placement', () => {
heartbeat(hostId, { ...inventory(), memoryTotal: 3 * GiB, memoryAvailable: 2 * GiB,
instanceResources: evidence === 'missing' ? [] : [{ instanceId: instance.id, generation: instance.generation + 1,
uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict',
message: expect.stringContaining('3.00 GiB projected, 2.53 GiB usable') });
const next = queueNativeCreation(input);
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
await advance();
expect(await operationFor(next.deploymentId)).toBeNull();
expect((await detail(next.deploymentId)).stage).toContain('3.00 GiB projected, 2.53 GiB usable');
});
it('waits when capacity disappears after admission without sending a create command', async () => {
await prepareLibrary();
@@ -931,11 +1026,16 @@ describe('automatic capacity admission and placement', () => {
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
const result = queueNativeCreation(input);
expect(result.ok).toBe(true); if (!result.ok) return;
expect(nativeInstance(result.name)).toBeNull();
await advance();
expect(nativeInstance(result.name)?.host_id).toBe(other.id);
updateHost(other.id, { mode: 'draining' });
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
const next = queueNativeCreation(input);
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
await advance();
expect(await operationFor(next.deploymentId)).toBeNull();
});
it('spreads collection admission across hosts, allowing the same port on different hosts', async () => {
it('spreads collection dispatch across hosts, allowing the same port on different hosts', async () => {
await prepareLibrary(); await secondHost({ cpuCount: 1, cpuUsagePercent: 0 });
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
@@ -944,10 +1044,17 @@ describe('automatic capacity admission and placement', () => {
const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
const preview = previewCollection(blueprint.id);
expect(preview.ready).toBe(true);
expect(preview.servers.map(server => server.port)).toEqual([28960, 28960]);
expect(new Set(preview.servers.map(server => server.native?.hostId)).size).toBe(2);
expect(preview.servers.map(server => server.port)).toEqual([null, null]);
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) }).ok).toBe(true);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
await advance();
// Concurrent ranking can select the same host before the first
// reservation commits. The held job retries against the new usage.
await advance();
expect(db().prepare('SELECT DISTINCT host_id FROM native_instances').all()).toHaveLength(2);
expect(db().prepare('SELECT port FROM native_instances').all()).toEqual([{ port: 28960 }, { port: 28960 }]);
});
});