Automatically place servers using learned resource capacity
This commit is contained in:
@@ -13,7 +13,7 @@ vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async ()
|
||||
|
||||
let fixture: Fixture; let hostId: string; let libraryId: string;
|
||||
const image = 'example.test/plutainer:test';
|
||||
const inventory = () => ({ protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' }] : [] });
|
||||
const inventory = () => ({ cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] });
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = '';
|
||||
fixture = await newFixture([]);
|
||||
@@ -610,3 +610,182 @@ describe('native instance provisioning', () => {
|
||||
expect(getServerKey(job.name)).toBe('saved-key');
|
||||
});
|
||||
});
|
||||
|
||||
describe('automatic capacity admission and placement', () => {
|
||||
const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'key' };
|
||||
async function secondHost(overrides = {}) {
|
||||
const api = await import('../../src/lib/server/hosts/index.js');
|
||||
const libs = await import('../../src/lib/server/game-files/index.js');
|
||||
const host = api.createHost({ name: 'Second host', publicAddress: 'second.example', portMin: 28960, portMax: 28970 });
|
||||
api.updateHost(host.id, { publicAddress: 'second.example' });
|
||||
api.enrollHost(api.createEnrollment(host.id).token, { ...inventory(), ...overrides });
|
||||
const version = libs.getLibraryVersion(libraryId)!;
|
||||
libs.updateHostLibraryCaches(host.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: version.manifestSha256 }]);
|
||||
return host;
|
||||
}
|
||||
it('admits without a host, then selects the least loaded eligible host and its network defaults', async () => {
|
||||
await prepareLibrary();
|
||||
const other = await secondHost({ cpuUsagePercent: 0 });
|
||||
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const result = queueCreateServer(input);
|
||||
expect(result.ok).toBe(true); if (!result.ok) return;
|
||||
expect(nativeInstance(result.name)?.host_id).toBe(hostId); // Fast first fit.
|
||||
await advance();
|
||||
expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id);
|
||||
expect(nativeInstance(result.name)).toMatchObject({ host_id: other.id, public_address: 'second.example' });
|
||||
expect(recipeForDeployment(result.deploymentId)?.recipe.native?.hostId).toBe(other.id);
|
||||
// Once submitted, even a now-offline host cannot trigger another placement.
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(other.id);
|
||||
await advance();
|
||||
expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id);
|
||||
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
|
||||
});
|
||||
it('moves an unsubmitted reservation off an offline host and preserves explicit network overrides', async () => {
|
||||
await prepareLibrary(); await secondHost();
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
const result = queueNativeCreation({ ...input, publicAddress: 'nat.example', publicPort: 30000 });
|
||||
expect(result.ok).toBe(true); if (!result.ok) return;
|
||||
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId);
|
||||
await advance();
|
||||
expect(nativeInstance(result.name)).toMatchObject({ public_address: 'nat.example', public_port: 30000 });
|
||||
expect((await operationFor(result.deploymentId))?.hostId).not.toBe(hostId);
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toEqual([]);
|
||||
});
|
||||
it.each([
|
||||
{ cpuUsagePercent: undefined }, { memoryAvailable: undefined }, { cpuCount: 0 },
|
||||
{ cpuUsagePercent: 99 }, { memoryAvailable: 1024 ** 3 }
|
||||
])('rejects admission with unavailable resources: %j', async overrides => {
|
||||
await prepareLibrary();
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(hostId, { ...inventory(), ...overrides });
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict', message: expect.stringContaining('capacity') });
|
||||
expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
||||
});
|
||||
it('rechecks preview admission without binding the request to the preview host or port', async () => {
|
||||
await prepareLibrary();
|
||||
const other = await secondHost();
|
||||
const { updateHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
updateHost(other.id, { portMin: 30000, portMax: 30002 });
|
||||
const { previewNativeCreation, queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const preview = previewNativeCreation(input);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId);
|
||||
const result = queueNativeCreation({ ...input, previewId: preview.id });
|
||||
expect(result).toMatchObject({ ok: true, name: preview.name, port: 30000 });
|
||||
expect(queueNativeCreation({ ...input, previewId: preview.id })).toEqual(result);
|
||||
});
|
||||
it('rejects an entire collection when capacity shrinks after its preview', async () => {
|
||||
await prepareLibrary();
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
|
||||
const preview = previewCollection(blueprint.id);
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
|
||||
expect(() => queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) })).toThrow('capacity');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
|
||||
});
|
||||
it('uses calibrated footprints to admit more instances than the cold-start allowance', async () => {
|
||||
const first = await queue();
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
const { recordInstanceResources, capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const measured = { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 };
|
||||
heartbeat(hostId, measured);
|
||||
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
|
||||
const instance = nativeInstance(first.name)!;
|
||||
for (let minute = 0; minute < 25; minute++) {
|
||||
recordInstanceResources(hostId, { ...measured, instanceResources: [{ instanceId: instance.id, generation: 1,
|
||||
uptimeSeconds: minute < 4 ? minute * 60 : 600, cpuCores: 0.1, memoryBytes: 256 * 1024 ** 2 }] },
|
||||
new Date(Date.now() - (25 - minute) * 60_000).toISOString());
|
||||
}
|
||||
expect(capacityCalibration(hostId).startup.source).toBe('host');
|
||||
expect(queueNativeCreation(input)).toMatchObject({ ok: true });
|
||||
});
|
||||
it('charges queued reservations and releases them on cancellation', async () => {
|
||||
await prepareLibrary();
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
const first = queueNativeCreation(input);
|
||||
expect(first.ok).toBe(true); if (!first.ok) return;
|
||||
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
|
||||
cancelCreation(first.deploymentId);
|
||||
expect(queueNativeCreation(input)).toMatchObject({ ok: true });
|
||||
});
|
||||
it('waits when capacity disappears after admission without sending a create command', async () => {
|
||||
await prepareLibrary();
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const first = queueNativeCreation(input);
|
||||
expect(first.ok).toBe(true); if (!first.ok) return;
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 });
|
||||
await advance();
|
||||
expect(await operationFor(first.deploymentId)).toBeNull();
|
||||
expect((await detail(first.deploymentId)).stage).toContain('capacity');
|
||||
});
|
||||
it('uses recent history at placement while keeping admission to current inventory', async () => {
|
||||
await prepareLibrary();
|
||||
const other = await secondHost({ cpuUsagePercent: 30 });
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available)
|
||||
VALUES(?,?,16,95,?,?)`).run(hostId, new Date(Date.now() - 60_000).toISOString(), 64 * 1024 ** 3, 60 * 1024 ** 3);
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const first = queueNativeCreation(input);
|
||||
expect(first.ok).toBe(true); if (!first.ok) return;
|
||||
await advance();
|
||||
expect((await operationFor(first.deploymentId))?.hostId).toBe(other.id);
|
||||
});
|
||||
it('persists nullable CPU/memory samples and prunes old history per host', async () => {
|
||||
await prepareLibrary();
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
db().prepare("INSERT INTO ec_host_resource_samples(host_id,sampled_at) VALUES(?,'2000-01-01')").run(hostId);
|
||||
heartbeat(hostId, { ...inventory(), cpuUsagePercent: undefined });
|
||||
closeDb();
|
||||
const rows = db().prepare('SELECT * FROM ec_host_resource_samples WHERE host_id=? ORDER BY sampled_at').all(hostId) as { cpu_usage_percent: number | null; memory_available: number; sampled_at: string }[];
|
||||
expect(rows.at(-1)).toMatchObject({ cpu_usage_percent: null, memory_available: 60 * 1024 ** 3 });
|
||||
expect(rows.some(row => row.sampled_at === '2000-01-01')).toBe(false);
|
||||
});
|
||||
it('checks both port protocols and skips draining hosts', async () => {
|
||||
await prepareLibrary();
|
||||
const other = await secondHost();
|
||||
const { reserveHostPort, updateHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
for (let port = 28960; port <= 28970; port++) reserveHostPort(hostId, 'external-' + port, { port, protocol: 'tcp' });
|
||||
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
||||
const result = queueNativeCreation(input);
|
||||
expect(result.ok).toBe(true); if (!result.ok) return;
|
||||
expect(nativeInstance(result.name)?.host_id).toBe(other.id);
|
||||
updateHost(other.id, { mode: 'draining' });
|
||||
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
|
||||
});
|
||||
it('spreads collection admission across hosts, allowing the same port on different hosts', async () => {
|
||||
await prepareLibrary(); await secondHost({ cpuCount: 1, cpuUsagePercent: 0 });
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
|
||||
const preview = previewCollection(blueprint.id);
|
||||
expect(preview.ready).toBe(true);
|
||||
expect(preview.servers.map(server => server.port)).toEqual([28960, 28960]);
|
||||
expect(new Set(preview.servers.map(server => server.native?.hostId)).size).toBe(2);
|
||||
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) }).ok).toBe(true);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT DISTINCT host_id FROM native_instances').all()).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user