Pool Plutonium keys by game for launches and autoscaling
This commit is contained in:
@@ -12,11 +12,12 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
|
||||
|
||||
let queuedKeys = 0;
|
||||
let fixture: Fixture; let hostId: string; let libraryId: string;
|
||||
const image = 'example.test/plutainer:test';
|
||||
const inventory = () => ({ cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] });
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = '';
|
||||
vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = ''; queuedKeys = 0;
|
||||
fixture = await newFixture([]);
|
||||
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
|
||||
vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password');
|
||||
@@ -48,7 +49,7 @@ async function queue(game: 't5' | 't6' = 't5', placement: 'pinned' | 'automatic'
|
||||
await prepareLibrary(game); remote.map = game === 't5' ? 'zombie_theater' : 'zm_transit';
|
||||
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const result = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town',
|
||||
hostId: placement === 'pinned' ? hostId : undefined, libraryId, image, plutoKey: 'saved-key' });
|
||||
hostId: placement === 'pinned' ? hostId : undefined, libraryId, image, plutoKey: ++queuedKeys === 1 ? 'saved-key' : `saved-key-${queuedKeys}` });
|
||||
if (!result.ok) throw new Error(result.message);
|
||||
return result;
|
||||
}
|
||||
@@ -990,8 +991,9 @@ describe('native instance provisioning', () => {
|
||||
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind LIKE 'instance.%'").get()).toEqual({ count: 0 });
|
||||
const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js');
|
||||
expect(getLibraryVersion(libraryId)?.state).toBe('ready');
|
||||
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
expect(getServerKey(job.name)).toBe('saved-key');
|
||||
const { getServerKey, serverKeyPool } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
expect(getServerKey(job.name)).toBeUndefined();
|
||||
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1083,7 +1085,7 @@ describe('automatic capacity admission and placement', () => {
|
||||
const preview = previewCollection(blueprint.id);
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 });
|
||||
const result = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) });
|
||||
const result = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) });
|
||||
expect(result).toMatchObject({ ok: true, collectionId: expect.any(String) });
|
||||
await advance();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
@@ -1117,7 +1119,7 @@ describe('automatic capacity admission and placement', () => {
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
const measured = { ...inventory(), cpuCount: 1, cpuUsagePercent: 50 };
|
||||
heartbeat(hostId, measured);
|
||||
const next = queueNativeCreation(input);
|
||||
const next = queueNativeCreation({ ...input, plutoKey: 'next-key' });
|
||||
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
|
||||
await advance();
|
||||
expect(await operationFor(next.deploymentId)).toBeNull();
|
||||
@@ -1151,7 +1153,7 @@ describe('automatic capacity admission and placement', () => {
|
||||
const blueprint = saveBlueprint({ name: 'Staggered lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 3 }] });
|
||||
const preview = previewCollection(blueprint.id);
|
||||
expect(preview.ready).toBe(true);
|
||||
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) }).ok).toBe(true);
|
||||
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }).ok).toBe(true);
|
||||
await advance();
|
||||
const creates = () => db().prepare("SELECT instance_id FROM ec_host_operations WHERE kind='instance.create'").all() as { instance_id: string }[];
|
||||
expect(creates()).toHaveLength(1);
|
||||
@@ -1170,7 +1172,7 @@ describe('automatic capacity admission and placement', () => {
|
||||
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
const first = queueNativeCreation(input);
|
||||
expect(first.ok).toBe(true); if (!first.ok) return;
|
||||
const second = queueNativeCreation(input);
|
||||
const second = queueNativeCreation({ ...input, plutoKey: 'second-key' });
|
||||
expect(second).toMatchObject({ ok: true, name: 't5_zombie_theater_2', port: null });
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
|
||||
@@ -1191,7 +1193,7 @@ describe('automatic capacity admission and placement', () => {
|
||||
(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,4,?,?,?)`)
|
||||
.run(hostId, new Date(Date.now() - minute * 60_000).toISOString(), resource === 'cpu' ? 100 : 10,
|
||||
4 * GiB, resource === 'memory' ? GiB / 8 : Math.floor(3.8 * GiB));
|
||||
const jobs = Array.from({ length: 3 }, () => queueNativeCreation(input));
|
||||
const jobs = Array.from({ length: 3 }, (_, index) => queueNativeCreation({ ...input, plutoKey: `key-${index}` }));
|
||||
expect(jobs.every(job => job.ok)).toBe(true);
|
||||
await advance();
|
||||
expect(await operationFor(jobs[0].ok ? jobs[0].deploymentId : 'missing')).not.toBeNull();
|
||||
@@ -1205,7 +1207,7 @@ describe('automatic capacity admission and placement', () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const GiB = 1024 ** 3;
|
||||
heartbeat(hostId, { ...inventory(), cpuCount: 4, memoryTotal: 4 * GiB, memoryAvailable: Math.floor(3.8 * GiB) });
|
||||
const jobs = Array.from({ length: 3 }, () => queueNativeCreation(input));
|
||||
const jobs = Array.from({ length: 3 }, (_, index) => queueNativeCreation({ ...input, plutoKey: `key-${index}` }));
|
||||
expect(jobs.every(job => job.ok)).toBe(true);
|
||||
// Learning increases the startup allowance after admission. All three
|
||||
// no longer fit concurrently, but this otherwise empty host fits one.
|
||||
@@ -1244,7 +1246,7 @@ describe('automatic capacity admission and placement', () => {
|
||||
// Adding a new 1 GiB server fits below 2.53 GiB usable memory.
|
||||
heartbeat(hostId, { ...host, memoryAvailable: 2 * GiB, instanceResources: [{ instanceId: instance.id,
|
||||
generation: instance.generation, uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
|
||||
const second = queueNativeCreation(input);
|
||||
const second = queueNativeCreation({ ...input, plutoKey: 'second-key' });
|
||||
expect(second).toMatchObject({ ok: true });
|
||||
await advance();
|
||||
expect(await operationFor(second.ok ? second.deploymentId : 'missing')).not.toBeNull();
|
||||
@@ -1265,7 +1267,7 @@ describe('automatic capacity admission and placement', () => {
|
||||
heartbeat(hostId, { ...inventory(), memoryTotal: 3 * GiB, memoryAvailable: 2 * GiB,
|
||||
instanceResources: evidence === 'missing' ? [] : [{ instanceId: instance.id, generation: instance.generation + 1,
|
||||
uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
|
||||
const next = queueNativeCreation(input);
|
||||
const next = queueNativeCreation({ ...input, plutoKey: 'next-key' });
|
||||
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
|
||||
await advance();
|
||||
expect(await operationFor(next.deploymentId)).toBeNull();
|
||||
@@ -1318,7 +1320,7 @@ describe('automatic capacity admission and placement', () => {
|
||||
await advance();
|
||||
expect(nativeInstance(result.name)?.host_id).toBe(other.id);
|
||||
updateHost(other.id, { mode: 'draining' });
|
||||
const next = queueNativeCreation(input);
|
||||
const next = queueNativeCreation({ ...input, plutoKey: 'next-key' });
|
||||
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
|
||||
await advance();
|
||||
expect(await operationFor(next.deploymentId)).toBeNull();
|
||||
@@ -1333,7 +1335,7 @@ describe('automatic capacity admission and placement', () => {
|
||||
const preview = previewCollection(blueprint.id);
|
||||
expect(preview.ready).toBe(true);
|
||||
expect(preview.servers.map(server => server.port)).toEqual([null, null]);
|
||||
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) }).ok).toBe(true);
|
||||
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }).ok).toBe(true);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
|
||||
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
||||
@@ -1481,6 +1483,7 @@ describe('cancelled native mutation bookkeeping', () => {
|
||||
it.each(['rebuild','destroy'] as const)('reconciles a delivered %s receipt without resuming the cancelled job', async action => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
|
||||
@@ -1491,6 +1494,7 @@ describe('cancelled native mutation bookkeeping', () => {
|
||||
await advance(); const operation = (await operationFor(id))!;
|
||||
pendingHostOperations(hostId);
|
||||
expect(cancelProvisioningJob(id)).toBe(true);
|
||||
expect(serverKeyPool('t5').available).toBe(0);
|
||||
const replacement = action === 'rebuild' ? requestLifecycle(job.name, 'restart') : null;
|
||||
if (replacement && !replacement.ok) throw new Error(replacement.error);
|
||||
await advance();
|
||||
@@ -1498,6 +1502,8 @@ describe('cancelled native mutation bookkeeping', () => {
|
||||
await advance(); await advance();
|
||||
if (replacement?.ok) expect(await operationFor(replacement.deploymentId)).toMatchObject({ kind: 'instance.restart', generation: 2 });
|
||||
expect((await detail(id)).state).toBe('cancelled');
|
||||
expect(serverKeyPool('t5').available).toBe(action === 'destroy' ? 1 : 0);
|
||||
expect(getServerKey(job.name)).toBe(action === 'destroy' ? undefined : 'saved-key');
|
||||
if(action === 'rebuild') expect(db().prepare('SELECT generation,retired_at FROM native_instances WHERE server=?').get(job.name)).toEqual({generation:2,retired_at:null});
|
||||
else {
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
||||
@@ -1508,6 +1514,7 @@ describe('cancelled native mutation bookkeeping', () => {
|
||||
it.each([false,true])('cancels a host migration without restarting it or leaking destination reservations (delivered=%s)', async delivered => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const { serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
||||
@@ -1521,12 +1528,15 @@ describe('cancelled native mutation bookkeeping', () => {
|
||||
expect(migration.remove_operation_id).toBeTruthy();
|
||||
if(delivered) hosts.pendingHostOperations(hostId);
|
||||
expect(cancelProvisioningJob(migration.deployment_id)).toBe(true);
|
||||
expect(serverKeyPool('t5').available).toBe(0);
|
||||
if(delivered) hosts.reportHostOperation(hostId,migration.remove_operation_id,{state:'completed'});
|
||||
await processProvisioning(); await processProvisioning();
|
||||
expect((await detail(migration.deployment_id)).state).toBe('cancelled');
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(destination.id)).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM ec_host_migrations').all()).toHaveLength(1);
|
||||
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toHaveLength(delivered ? 0 : 2);
|
||||
expect(serverKeyPool('t5').available).toBe(delivered ? 1 : 0);
|
||||
expect(getServerKey(job.name)).toBe(delivered ? undefined : 'saved-key');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1547,3 +1557,73 @@ describe('additional concurrency admission', () => {
|
||||
expect(await operationFor(second.deploymentId)).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('game key pool launch lifecycle', () => {
|
||||
it('reserves across maps atomically, rejects duplicate credentials, and reuses a cancelled launch key', async () => {
|
||||
await prepareLibrary();
|
||||
const { saveServerKey, serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { queueCreateServer, destroyServer, previewCreation } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
saveServerKey('t5', 'shared-t5-key', 'Any T5 map');
|
||||
saveServerKey('t6', 'shared-t6-key');
|
||||
const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'moon', hostId, libraryId, image };
|
||||
expect(previewCreation(input).hasSavedKey).toBe(true);
|
||||
const first = queueCreateServer(input);
|
||||
if (!first.ok) throw new Error(first.message);
|
||||
expect(first.name).toBe('t5_zombie_moon_1');
|
||||
expect(getServerKey(first.name)).toBe('shared-t5-key');
|
||||
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 0 });
|
||||
expect(serverKeyPool('t6')).toEqual({ total: 1, available: 1 });
|
||||
expect(previewCreation({ ...input, mapSlug: 'theater' }).hasSavedKey).toBe(false);
|
||||
expect(queueCreateServer({ ...input, mapSlug: 'theater' }).ok).toBe(false);
|
||||
expect(queueCreateServer({ ...input, mapSlug: 'theater', plutoKey: 'shared-t5-key' }).ok).toBe(false);
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
expect(serverKeyPool('t5').available).toBe(0);
|
||||
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1);
|
||||
expect(destroyServer(first.name).ok).toBe(true);
|
||||
expect(serverKeyPool('t5').available).toBe(1);
|
||||
const second = queueCreateServer({ ...input, mapSlug: 'theater' });
|
||||
if (!second.ok) throw new Error(second.message);
|
||||
expect(getServerKey(second.name)).toBe('shared-t5-key');
|
||||
expect(getServerKey(first.name)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('keeps a key through stop and rebuild, then releases it only after confirmed removal', async () => {
|
||||
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
||||
const { serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const stopped = requestLifecycle(job.name, 'stop');
|
||||
if (!stopped.ok) throw new Error(stopped.error);
|
||||
await advance(); remote.state = 'stopped'; await acknowledge(stopped.deploymentId);
|
||||
expect(serverKeyPool('t5').available).toBe(0);
|
||||
const rebuilt = requestLifecycle(job.name, 'rebuild');
|
||||
if (!rebuilt.ok) throw new Error(rebuilt.error);
|
||||
await advance(); remote.state = 'running'; await acknowledge(rebuilt.deploymentId);
|
||||
expect(getServerKey(job.name)).toBe('saved-key');
|
||||
expect(serverKeyPool('t5').available).toBe(0);
|
||||
const removal = destroyServer(job.name);
|
||||
if (!removal.ok || !removal.deploymentId) throw new Error('Expected removal');
|
||||
await advance();
|
||||
expect(serverKeyPool('t5').available).toBe(0);
|
||||
const operation = (await operationFor(removal.deploymentId))!;
|
||||
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
||||
reportHostOperation(hostId, operation.id, { state: 'unknown', error: 'Receipt lost' });
|
||||
await advance();
|
||||
expect(serverKeyPool('t5').available).toBe(0);
|
||||
await acknowledge(removal.deploymentId);
|
||||
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
|
||||
expect(getServerKey(job.name)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('keeps the reservation for a submitted cancellation until the host confirms cleanup', async () => {
|
||||
const job = await queue(); await advance();
|
||||
const { serverKeyPool } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
||||
const cleanup = cancelCreation(job.deploymentId)!;
|
||||
expect(cleanup).toEqual(expect.any(String));
|
||||
expect(serverKeyPool('t5').available).toBe(0);
|
||||
await acknowledge(cleanup);
|
||||
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user