josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Pool Plutonium keys by game for launches and autoscaling

Browse Source
This commit is contained in:
josh
2026-10-07 11:24:53 -04:00
parent 56e5a2d06d
commit ec45371478
47 changed files with 956 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/autoscaling-delete.test.ts
+2 -2
View File
@@ -10,7 +10,7 @@ async function setup(mode = 'dry-run') {
if (mode.startsWith('create')) await prepareNativeHost();
const { db } = await import('../../src/lib/server/db/index.js');
const scaling = await import('../../src/lib/server/autoscaling/index.js');
db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run('t6_zombie_town_1', 'test-key');
db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('key-1','t6','test-key')").run();
const input = { ...settings, mode };
const saved = scaling.saveScaling(input, undefined, undefined, scaling.previewScaling(input).token);
return { db, scaling, saved, input };
@@ -20,7 +20,7 @@ it('deletes policy evidence atomically while preserving lobbies, keys, and unrel
const { db, scaling, saved } = await setup();
db().prepare("INSERT INTO servers(name,game,map_slug,map) VALUES('t6_zombie_town_1','t6','town','zm_transit')").run();
await scaling.runAutoscaling();
db().prepare("INSERT INTO server_keys(server_id,pluto_key) VALUES('t5_zombie_theater_1','other-key')").run();
db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('other-key','t5','other-key')").run();
const otherSettings = { ...settings, game: 't5', mapSlug: 'theater', mode: 'off' };
const other = scaling.saveScaling(otherSettings, undefined, undefined, scaling.previewScaling(otherSettings).token);
for (const table of ['autoscaling_decisions', 'autoscaling_members', 'autoscaling_samples']) expect(db().prepare(`SELECT COUNT(*) AS n FROM ${table} WHERE policy_id=?`).get(saved.id)).toEqual({ n: 1 });
tests/unit/autoscaling-dry-run.test.ts
+6 -4
View File
@@ -6,7 +6,7 @@ let fixture: Fixture;
beforeEach(async () => {
vi.resetModules(); fixture = await newFixture([]);
const { db } = await import('../../src/lib/server/db/index.js');
for (const n of [3, 8, 9, 10]) db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(`t6_zombie_town_${n}`, `secret-${n}`);
for (const n of [3, 8, 9, 10]) db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(`key-${n}`, `secret-${n}`);
});
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); });
const settings = { game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' };
@@ -22,7 +22,7 @@ it('persists deduplicated dry-run decisions across restart without provisioning
const view = scaling.scalingReport().policies[0];
expect(view.policy.id).toBe(saved.id);
expect(view.history).toHaveLength(1);
expect(view.decision).toMatchObject({ action: 'create', server: 't6_zombie_town_3' });
expect(view.decision).toMatchObject({ action: 'create', server: 't6_zombie_town_1' });
expect(JSON.stringify(view)).not.toContain('secret');
for (const table of ['server_deployments', 'native_creation_intents', 'native_instances']) expect(db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).toEqual({ n: 0 });
});
@@ -35,8 +35,10 @@ it('rejects duplicate policies, stale edits and changed previews', async () => {
expect(() => scaling.previewScaling(settings, saved.id, 0)).toThrow('changed');
expect(() => scaling.saveScaling({ ...settings, maximum: 5 }, saved.id, saved.revision, preview.token)).toThrow('preview changed');
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_10'").run();
expect(() => scaling.saveScaling({ ...settings, mode: 'create-destroy' }, saved.id, saved.revision, scaling.previewScaling({ ...settings, mode: 'create-destroy' }, saved.id, saved.revision).token)).toThrow('Save keys');
const beforeKeyLoss = scaling.previewScaling(settings, saved.id, saved.revision);
db().prepare("DELETE FROM server_keys WHERE id='key-10'").run();
expect(() => scaling.saveScaling(settings, saved.id, saved.revision, beforeKeyLoss.token)).toThrow('preview changed');
expect(scaling.saveScaling(settings, saved.id, saved.revision, scaling.previewScaling(settings, saved.id, saved.revision).token).maximum).toBe(4);
});
it('pauses evaluations and prunes expired observations without deleting current history', async () => {
tests/unit/autoscaling-game.test.ts
+12 -8
View File
@@ -14,7 +14,7 @@ async function setup() {
await prepareNativeHost();
const { db } = await import('../../src/lib/server/db/index.js');
const scaling = await import('../../src/lib/server/autoscaling/index.js');
for (const map of maps) for (const n of [1, 2]) db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(`t6_zombie_${map.slug}_${n}`, `key-${map.slug}-${n}`);
db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('key-1','t6','game-key')").run();
return { db, scaling };
}
@@ -24,7 +24,7 @@ it('applies targets and spare deficits independently to every map', () => {
const members: ScalingMember[] = maps.flatMap(map => [1, 2].map(n => ({ name: `t6_zombie_${map.slug}_${n}`, mapSlug: map.slug,
identity: `${map.slug}:${n}`, state: map.slug === 'town' ? 'occupied' : 'empty', players: map.slug === 'town' ? 1 : 0,
observedAt: now, source: 'rcon', emptySince: 1, blocker: null, operationId: null })));
const snapshot = { members, reservedNames: members.map(m => m.name), keyedNames: members.map(m => m.name), blockers: [] };
const snapshot = { members, reservedNames: members.map(m => m.name), keyPool: { total: 15, available: 1 }, blockers: [] };
// Other maps' empty lobbies cannot satisfy Town's spare target or maximum.
expect(planScaling(policy, snapshot, now)).toMatchObject({ action: 'destroy', server: 't6_zombie_transit_1', desired: maps.length + 1, deficit: 1 });
snapshot.members = members.filter(m => m.name !== 't6_zombie_town_2');
@@ -37,9 +37,9 @@ it('applies targets and spare deficits independently to every map', () => {
it('persists a game rule, queues the selected map recipe, and protects every map', async () => {
const { db, scaling } = await setup();
db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run('t5_zombie_theater_1', 'other-game');
db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('other-key','t5','other-game')").run();
const preview = scaling.previewScaling(settings);
expect(preview.snapshot.keyedNames).toHaveLength(maps.length * 2);
expect(preview.snapshot.keyPool).toEqual({ total: 1, available: 1 });
const saved = scaling.saveScaling(settings, undefined, undefined, preview.token);
expect(() => scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token)).toThrow();
expect(() => scaling.previewScaling({ ...settings, mapSlug: 'town' }, saved.id, saved.revision)).toThrow('cannot change');
@@ -64,10 +64,14 @@ it('persists a game rule, queues the selected map recipe, and protects every map
expect(scaling.previewScaling(settings, saved.id, current.revision).blockers.join(' ')).toContain(mapPolicy.id);
});
it('requires live key coverage on each map even when the game has enough keys in total', async () => {
it('holds game-wide growth when its shared pool is exhausted by another map', async () => {
const { db, scaling } = await setup();
db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_2'").run();
db().prepare("INSERT INTO server_key_leases(server_id,key_id) VALUES('t6_zombie_town_2','key-1')").run();
const preview = scaling.previewScaling(settings);
expect(preview.blockers.join(' ')).toContain('Town Survival');
expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('Town Survival');
expect(preview.snapshot.keyPool).toEqual({ total: 1, available: 0 });
expect(preview.blockers).toEqual([]);
expect(preview.decision).toMatchObject({ action: 'wait', reason: expect.stringContaining('keys are in use') });
scaling.saveScaling(settings, undefined, undefined, preview.token);
expect(await scaling.runAutoscaling()).toBe(false);
expect(db().prepare('SELECT COUNT(*) AS n FROM server_creation_jobs').get()).toEqual({ n: 0 });
});
tests/unit/autoscaling-keys.test.ts
+73 -57
View File
@@ -1,7 +1,6 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { prepareNativeHost } from '../helpers/native-host.js';
import { zombieMaps } from '../../src/lib/map-aliases.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
@@ -10,73 +9,90 @@ const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'dry-run' };
async function setup() {
const { db } = await import('../../src/lib/server/db/index.js');
const scaling = await import('../../src/lib/server/autoscaling/index.js');
const key = (name: string, value = `key-${name}`) => db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(name, value);
return { db, scaling, key };
const { saveServerKey, reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
return { db, scaling, saveServerKey, reserveServerKey };
}
it.each(['off', 'dry-run', 'create-only', 'create-destroy'])('rejects new %s policies without enough keys through both service and API', async mode => {
const { db, scaling, key } = await setup();
key('t6_zombie_town_3');
const input = { ...settings, mode };
it.each(['off', 'dry-run', 'create-only', 'create-destroy'])('saves a %s policy without preallocating keys for its maximum', async mode => {
if (mode.startsWith('create')) await prepareNativeHost();
const { db, scaling } = await setup();
const input = { ...settings, mode, maximum: 20 };
const preview = scaling.previewScaling(input);
expect(preview.blockers[0]).toContain('1 saved; 1 more required');
expect(() => scaling.saveScaling(input, undefined, undefined, preview.token)).toThrow('Save keys');
expect(preview.blockers).toEqual([]);
expect(preview.decision.action).toBe('wait');
if (mode !== 'off') expect(preview.decision.reason).toContain('No Plutonium keys');
const { POST } = await import('../../src/routes/api/autoscaling/+server.js');
const response = await POST({ request: new Request('http://localhost/api/autoscaling', { method: 'POST', body: JSON.stringify({ action: 'save', settings: input, token: preview.token }) }) });
expect(response.status).toBe(400);
expect(await response.json()).toMatchObject({ error: expect.stringContaining('Save keys') });
expect(db().prepare('SELECT COUNT(*) AS n FROM autoscaling_policies').get()).toEqual({ n: 0 });
expect(response.status).toBe(200);
expect(db().prepare('SELECT COUNT(*) AS n FROM autoscaling_policies').get()).toEqual({ n: 1 });
expect(await scaling.runAutoscaling()).toBe(false);
});
it('counts only supported, nonblank identities in the selected map and accepts exact nonconsecutive coverage', async () => {
const { scaling, key } = await setup();
for (const name of ['t6_zombie_town_0', 't6_zombie_town_21', 't6_zombie_town_01', 't6_zombie_town', 't6_town_2', 't5_zombie_theater_1', 't6_zombie_farm_1']) key(name);
key('t6_zombie_town_2', ' ');
key('t6_zombie_town_3');
expect(scaling.previewScaling(settings).snapshot.keyedNames).toEqual(['t6_zombie_town_3']);
key('t6_zombie_town_8');
it('uses any free key for the selected game independently of its label or map', async () => {
const { scaling, saveServerKey } = await setup();
saveServerKey('t5', 'other-game-key', 'Town Lobby 1');
saveServerKey('t6', 'game-key', 'Farm Lobby 8');
const preview = scaling.previewScaling(settings);
expect(preview.blockers).toEqual([]);
expect(preview.snapshot.keyPool).toEqual({ total: 1, available: 1 });
expect(preview.decision).toMatchObject({ action: 'create', server: 't6_zombie_town_1' });
expect(scaling.saveScaling(settings, undefined, undefined, preview.token).maximum).toBe(2);
});
it('requires coverage on every game map rather than pooling excess keys from another map', async () => {
const { scaling, key } = await setup();
for (const map of zombieMaps('t6')) for (const n of [3, 8]) if (map.slug !== 'town' || n === 3) key(`t6_zombie_${map.slug}_${n}`);
key('t6_zombie_farm_9');
const input = { ...settings, mapSlug: '*' };
it('uses one shared pool for every map without requiring a key for each maximum', async () => {
const { scaling, saveServerKey } = await setup();
saveServerKey('t6', 'game-key');
const input = { ...settings, mapSlug: '*', maximum: 20 };
const preview = scaling.previewScaling(input);
expect(preview.blockers).toEqual([expect.stringContaining('Town Survival')]);
expect(() => scaling.saveScaling(input, undefined, undefined, preview.token)).toThrow('Town Survival');
key('t6_zombie_town_8');
expect(scaling.saveScaling(input, undefined, undefined, scaling.previewScaling(input).token).mapSlug).toBe('*');
});
it('rechecks key deletion after preview and rejects increases beyond coverage', async () => {
const { db, scaling, key } = await setup();
key('t6_zombie_town_3'); key('t6_zombie_town_8');
const preview = scaling.previewScaling(settings);
db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_8'").run();
expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('preview changed');
expect(() => scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token)).toThrow('Save keys');
key('t6_zombie_town_8');
const saved = scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
const higher = { ...settings, maximum: 3 };
expect(() => scaling.saveScaling(higher, saved.id, saved.revision, scaling.previewScaling(higher, saved.id, saved.revision).token)).toThrow('Save keys');
expect(scaling.getScalingPolicy(saved.id)?.maximum).toBe(2);
});
it('allows an existing live policy to pause and withdraw pending work after all keys are removed', async () => {
await prepareNativeHost();
const { db, scaling, key } = await setup();
key('t6_zombie_town_3'); key('t6_zombie_town_8');
const live = { ...settings, mode: 'create-only' };
const saved = scaling.saveScaling(live, undefined, undefined, scaling.previewScaling(live).token);
await scaling.runAutoscaling();
db().prepare('DELETE FROM server_keys').run();
const off = { ...settings, mode: 'off' };
const preview = scaling.previewScaling(off, saved.id, saved.revision);
expect(preview.blockers).toEqual([]);
expect(scaling.saveScaling(off, saved.id, saved.revision, preview.token).mode).toBe('off');
expect(db().prepare("SELECT state FROM server_deployments WHERE action='create'").get()).toEqual({ state: 'cancelled' });
expect(preview.snapshot.keyPool).toEqual({ total: 1, available: 1 });
expect(scaling.saveScaling(input, undefined, undefined, preview.token).mapSlug).toBe('*');
});
it('rechecks game pool deletion and reservations after preview', async () => {
const { db, scaling, saveServerKey, reserveServerKey } = await setup();
const keyId = saveServerKey('t6', 'game-key');
const preview = scaling.previewScaling(settings);
db().prepare('DELETE FROM server_keys WHERE id=?').run(keyId);
expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('preview changed');
saveServerKey('t6', 'game-key');
const available = scaling.previewScaling(settings);
reserveServerKey('t6_zombie_farm_1', 't6');
expect(() => scaling.saveScaling(settings, undefined, undefined, available.token)).toThrow('preview changed');
const held = scaling.previewScaling(settings);
expect(held.decision.reason).toContain('keys are in use');
expect(held.snapshot.members).toEqual([]);
expect(scaling.saveScaling(settings, undefined, undefined, held.token).maximum).toBe(2);
});
it('shares capacity between live map policies and admits the job holding the last key', async () => {
await prepareNativeHost();
const { db, scaling, saveServerKey } = await setup();
saveServerKey('t6', 'game-key');
saveServerKey('t5', 'other-game-key');
for (const mapSlug of ['town', 'farm']) {
const input = { ...settings, mapSlug, mode: 'create-only' };
scaling.saveScaling(input, undefined, undefined, scaling.previewScaling(input).token);
}
expect(await scaling.runAutoscaling()).toBe(true);
const jobs = db().prepare('SELECT deployment_id,server FROM server_creation_jobs').all() as { deployment_id: string; server: string }[];
expect(jobs).toHaveLength(1);
expect(jobs[0].server).toBe('t6_zombie_farm_1');
const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
expect(scalingAdmission(jobs[0].deployment_id)).toBeNull();
const town = scaling.scalingReport().policies.find(p => p.policy.mapSlug === 'town')!;
expect(town.snapshot.keyPool).toEqual({ total: 1, available: 0 });
expect(town.decision.reason).toContain('keys are in use');
expect(town.policy.pausedReason).toBeNull();
expect(await scaling.runAutoscaling()).toBe(false);
expect(db().prepare('SELECT COUNT(*) AS n FROM server_creation_jobs').get()).toEqual({ n: 1 });
});
it('reserves a lobby identity for an unsettled failed launch with a retained key', async () => {
const { scaling, saveServerKey, reserveServerKey } = await setup();
saveServerKey('t6', 'held-key');
reserveServerKey('t6_zombie_town_1', 't6');
saveServerKey('t6', 'free-key');
const preview = scaling.previewScaling(settings);
expect(preview.snapshot.reservedNames).toEqual(['t6_zombie_town_1']);
expect(preview.decision).toMatchObject({ action: 'create', server: 't6_zombie_town_2' });
});
tests/unit/autoscaling-live.test.ts
+11 -9
View File
@@ -10,7 +10,7 @@ async function setup() {
await prepareNativeHost();
const { db } = await import('../../src/lib/server/db/index.js');
const scaling = await import('../../src/lib/server/autoscaling/index.js');
for (const n of [3, 8]) db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(`t6_zombie_town_${n}`, `key-${n}`);
for (const n of [3, 8]) db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(`key-${n}`, `key-${n}`);
const saved = scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
return { db, scaling, saved };
}
@@ -21,7 +21,7 @@ it('admits one durable keyed create and reuses it across ticks and restart', asy
await Promise.all([scaling.runAutoscaling(), scaling.runAutoscaling()]);
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
await scaling.runAutoscaling();
expect(db().prepare('SELECT server,state FROM server_deployments WHERE action=\'create\'').all()).toEqual([{ server: 't6_zombie_town_3', state: 'queued' }]);
expect(db().prepare('SELECT server,state FROM server_deployments WHERE action=\'create\'').all()).toEqual([{ server: 't6_zombie_town_1', state: 'queued' }]);
expect(db().prepare('SELECT COUNT(*) AS n FROM autoscaling_decisions WHERE resolved=0').get()).toEqual({ n: 1 });
const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string };
const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
@@ -30,16 +30,19 @@ it('admits one durable keyed create and reuses it across ticks and restart', asy
expect(getDeployment(id)?.sources).toContainEqual(expect.objectContaining({ kind: 'autoscaling', href: `/autoscaling#${saved.id}` }));
});
it('requires key coverage and withdraws an unsubmitted create when the policy pauses', async () => {
it('withdraws an unsubmitted create and returns its game key when the policy pauses', async () => {
const { db, scaling, saved } = await setup();
await scaling.runAutoscaling();
const off = { ...settings, mode: 'off' };
scaling.saveScaling(off, saved.id, saved.revision, scaling.previewScaling(off, saved.id, saved.revision).token);
expect(db().prepare("SELECT state FROM server_deployments WHERE action='create'").get()).toEqual({ state: 'cancelled' });
expect(db().prepare('SELECT owns_map FROM autoscaling_policies').get()).toEqual({ owns_map: 0 });
db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_8'").run();
expect(db().prepare('SELECT COUNT(*) AS n FROM server_key_leases').get()).toEqual({ n: 0 });
db().prepare('DELETE FROM server_keys').run();
const current = scaling.getScalingPolicy(saved.id)!;
expect(scaling.previewScaling(settings, saved.id, current.revision).blockers.join(' ')).toContain('Save keys');
const preview = scaling.previewScaling(settings, saved.id, current.revision);
expect(preview.blockers).toEqual([]);
expect(preview.snapshot.keyPool).toEqual({ total: 0, available: 0 });
});
it('blocks manual lifecycle and collection creation, including stale previews', async () => {
@@ -78,7 +81,7 @@ it('exact matching cannot remove an owned map absent from the collection', async
const { db } = await import('../../src/lib/server/db/index.js');
const at = new Date().toISOString();
db().prepare("INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,0)").run(town, at, at);
for (const n of [1, 2]) db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(`t6_zombie_town_${n}`, `key-${n}`);
for (const n of [1, 2]) db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(`key-${n}`, `key-${n}`);
const scaling = await import('../../src/lib/server/autoscaling/index.js');
scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
@@ -90,13 +93,12 @@ it('exact matching cannot remove an owned map absent from the collection', async
expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling');
});
it('rechecks evidence and global pause before dispatch', async () => {
it('honors global pause before dispatch while recognizing its own reserved key', async () => {
const { db, scaling } = await setup();
await scaling.runAutoscaling();
const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string };
const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_3'").run();
expect(scalingAdmission(id)).toContain('no longer eligible');
expect(scalingAdmission(id)).toBeNull();
scaling.setScalingGlobalPause(true);
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'cancelled' });
expect(await scaling.runAutoscaling()).toBe(false);
tests/unit/autoscaling-planner.test.ts
+8 -4
View File
@@ -7,7 +7,7 @@ const now = 2_000_000;
const policy: ScalingPolicy = { ...scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' }), id: 'policy', revision: 1, nextActionAt: 0, pausedReason: null, updatedAt: '' };
const name = (n: number) => `t6_zombie_town_${n}`;
const member = (n: number, state: ScalingMember['state'] = 'empty'): ScalingMember => ({ name: name(n), identity: `instance:${n}`, state, players: state === 'occupied' ? 1 : state === 'empty' ? 0 : null, observedAt: now, source: 'rcon', emptySince: 1, blocker: null, operationId: null });
const snapshot = (...members: ScalingMember[]): ScalingSnapshot => ({ members, reservedNames: members.map(m => m.name), keyedNames: [1, 2, 3, 4].map(name), blockers: [] });
const snapshot = (...members: ScalingMember[]): ScalingSnapshot => ({ members, reservedNames: members.map(m => m.name), keyPool: { total: 4, available: 4 - members.length }, blockers: [] });
describe('autoscaling policy planning', () => {
it('requires explicit bounded maximum and valid variant', () => {
@@ -19,9 +19,13 @@ describe('autoscaling policy planning', () => {
expect(planScaling(policy, snapshot(), now)).toMatchObject({ action: 'create', desired: 1 });
expect(planScaling(policy, snapshot(member(1, 'occupied'), member(2)), now).action).toBe('wait');
});
it('uses nonconsecutive keyed identities without moving keys', () => {
expect(planScaling(policy, { ...snapshot(), keyedNames: [name(8), name(3)] }, now).server).toBe(name(3));
expect(planScaling(policy, { ...snapshot(), keyedNames: [] }, now).reason).toContain('saved key');
it('selects the first free identity independently of the game key pool size', () => {
expect(planScaling(policy, { ...snapshot(), reservedNames: [name(1), name(3)], keyPool: { total: 1, available: 1 } }, now).server).toBe(name(2));
expect(planScaling(policy, { ...snapshot(), keyPool: { total: 0, available: 0 } }, now).reason).toContain('No Plutonium keys');
expect(planScaling(policy, { ...snapshot(), keyPool: { total: 2, available: 0 } }, now).reason).toContain('keys are in use');
});
it('still removes eligible surplus lobbies when the game pool is exhausted', () => {
expect(planScaling(policy, { ...snapshot(member(1), member(2)), keyPool: { total: 2, available: 0 } }, now)).toMatchObject({ action: 'destroy', server: name(1) });
});
it('holds for pending creation, unknown, stopped, or protected members', () => {
for (const state of ['creating', 'removing', 'unknown', 'stopped', 'protected'] as const) expect(planScaling(policy, snapshot(member(1, state)), now).action).toBe('wait');
tests/unit/autoscaling-removal.test.ts
+2 -2
View File
@@ -22,10 +22,10 @@ async function setup(gameWide = false) {
for (const name of servers) {
const map = zombieMaps('t6').find(m => name === `t6_zombie_${m.slug}_1` || name === `t6_zombie_${m.slug}_2`)!;
db().prepare("INSERT INTO servers(name,game,map,map_slug,container_state,last_players) VALUES(?,'t6',?,?,'running',0)").run(name, map.canonical, map.slug);
db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(name, `key-${name}`);
db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(`key-${name}`, `key-${name}`);
}
if (gameWide) for (const map of zombieMaps('t6').filter(m => m.slug !== 'town')) {
db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(`t6_zombie_${map.slug}_2`, `key-${map.slug}-2`);
db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(`key-${map.slug}-2`, `key-${map.slug}-2`);
}
const fresh = (offset: number) => {
vi.setSystemTime(base + offset); const at = new Date().toISOString();
tests/unit/autoscaling-snapshot.test.ts
+2 -2
View File
@@ -31,7 +31,7 @@ it('does not create schema mutations or host work when planning an empty map', a
const { db } = await import('../../src/lib/server/db/index.js');
const { scalingSnapshot } = await import('../../src/lib/server/autoscaling/snapshot.js');
const { planScaling } = await import('../../src/lib/server/autoscaling/planner.js');
db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run('t6_zombie_town_8', 'private-key');
expect(planScaling(policy, scalingSnapshot(policy), Date.now()).server).toBe('t6_zombie_town_8');
db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('key-8','t6','private-key')").run();
expect(planScaling(policy, scalingSnapshot(policy), Date.now()).server).toBe('t6_zombie_town_1');
expect(db().prepare('SELECT COUNT(*) AS n FROM server_deployments').get()).toEqual({ n: 0 });
});
tests/unit/beta-destruction.test.ts
+2 -2
View File
@@ -111,11 +111,11 @@ describe('active beta destruction guard', () => {
const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js');
const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
const { db } = await import('../../src/lib/server/db/index.js');
saveServerKey(server, 'saved-key');
reserveServerKey(server, 't5', 'saved-key');
const queued = action === 'destroy' ? destroyServer(server) : requestLifecycle(server, 'rebuild');
expect(queued.ok).toBe(true);
if (!queued.ok) throw new Error('Expected queued removal');
tests/unit/collection-launch-performance.test.ts
+1 -1
View File
@@ -117,6 +117,6 @@ describe('large native collection admission', () => {
.toThrow(change === 'newer game files' ? /game files changed/ : /host was not found/);
expect(db().prepare('SELECT id FROM server_launch_collections').all()).toEqual([]);
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([]);
expect(db().prepare('SELECT server_id FROM server_keys').all()).toEqual([]);
expect(db().prepare('SELECT id FROM server_keys').all()).toEqual([]);
});
});
tests/unit/deployment-collections.test.ts
+26 -8
View File
@@ -90,19 +90,36 @@ describe('multi-server blueprints', () => {
describe('atomic collection launches', () => {
it('counts available keys once per game across all maps without reserving them in a preview', async () => {
const { blueprint } = await previewLineup();
const { saveServerKey, reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const { db } = await import('../../src/lib/server/db/index.js');
saveServerKey('t5', 'pool-one'); saveServerKey('t5', 'pool-two'); saveServerKey('t6', 'pool-three');
const preview = previewCollection(blueprint.id);
expect(preview.servers.map(server => server.hasSavedKey)).toEqual([true, true, false, true, false]);
expect(db().prepare('SELECT * FROM server_key_leases').all()).toEqual([]);
reserveServerKey('t5_zombie_pentagon_1', 't5');
expect(() => queueCollection({ previewId: preview.id })).toThrow(/No available T5 server keys/);
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]);
expect(db().prepare('SELECT * FROM server_key_leases').all()).toHaveLength(1);
});
it('rolls back all earlier jobs and key changes if the final server has no valid key', async () => {
const { preview } = await previewLineup();
const { queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { saveServerKey, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
saveServerKey(preview.servers[0].name!, 'original-key');
saveServerKey('t5', 'original-key');
const keys = keysFor(preview);
delete keys[preview.servers[4].name!];
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/Save a Plutonium server key/);
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/No available .* server keys/);
for (const table of ['server_launch_collections', 'server_collection_members', 'server_creation_jobs', 'server_deployments', 'server_launch_previews', 'server_launch_recipes']) {
expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
}
expect(getServerKey(preview.servers[0].name!)).toBe('original-key');
expect(getServerKey(preview.servers[0].name!)).toBeUndefined();
expect(db().prepare('SELECT pluto_key FROM server_keys').get()).toEqual({ pluto_key: 'original-key' });
expect(db().prepare('SELECT * FROM server_key_leases').all()).toEqual([]);
expect(db().prepare('SELECT * FROM server_keys').all()).toHaveLength(1);
keys[preview.servers[4].name!] = 'invalid key';
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/without whitespace/);
@@ -115,7 +132,7 @@ describe('atomic collection launches', () => {
const { queueCollection, previewCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
for (const server of preview.servers) saveServerKey(server.name!, `saved-${server.name}`);
for (const server of preview.servers) saveServerKey(server.name!.slice(0, 2), `saved-${server.name}`);
const ready = previewCollection(blueprint.id);
expect(ready.servers.every(server => server.hasSavedKey)).toBe(true);
const queued = queueCollection({ previewId: ready.id });
@@ -171,17 +188,18 @@ describe('matching a blueprint exactly', () => {
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(first.deploymentId)).toEqual({ state: 'queued' });
expect(getCollection(queued.collectionId!)?.collection).toMatchObject({ done: 1, total: 1, active: 0 });
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
expect(getServerKey(second.name)).toBe('second');
expect(getServerKey(second.name)).toBeUndefined();
expect(db().prepare("SELECT pluto_key FROM server_keys WHERE pluto_key='second'").get()).toEqual({ pluto_key: 'second' });
});
it('rolls back removals and creation cancellations when a required key is missing', async () => {
it('rolls back removals and creation cancellations when the game pool is exhausted', async () => {
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
const old = queueCreateServer({ gameMode: 'zombie', mapSlug: 'moon', plutoKey: 'existing' });
if (!old.ok) throw new Error(old.message);
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 1 }]);
const plan = await blueprint([{ game: 't5', mapSlug: 'theater', count: 2 }]);
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const preview = previewCollection(plan.id, 'exact');
expect(() => queueCollection({ previewId: preview.id })).toThrow(/Save a Plutonium server key/);
expect(() => queueCollection({ previewId: preview.id })).toThrow(/No available .* server keys/);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(old.deploymentId)).toEqual({ state: 'queued' });
for (const table of ['server_collection_removals', 'server_launch_collections', 'server_creation_dependencies']) expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
tests/unit/host-deletion.test.ts
+1 -1
View File
@@ -153,7 +153,7 @@ describe('host deletion', () => {
{ hostId: other.id, automatic: false }
].entries()) {
expect(queueNativeCreationIntent({ server: `t5_zombie_theater_${index + 1}`, game: 't5', mapSlug: 'theater',
plutoKey: 'saved-key', selection: { ...selection, libraryId: 'library', image: 'plutainer:test' } }).ok).toBe(true);
plutoKey: `saved-key-${index}`, selection: { ...selection, libraryId: 'library', image: 'plutainer:test' } }).ok).toBe(true);
}
expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ allowed: true, blockers: [] });
expect(hosts.hostDeletionStatus(other.id).allowed).toBe(false);
tests/unit/host-preparation.test.ts
+1 -1
View File
@@ -353,7 +353,7 @@ describe('automatic host preparation', () => {
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
for (let index = 0; index < 2; index++) {
expect(queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', libraryId: old.id,
hostId: automatic ? undefined : id, image, plutoKey: 'saved-key' }).ok).toBe(true);
hostId: automatic ? undefined : id, image, plutoKey: `saved-key-${index}` }).ok).toBe(true);
}
// Also cover pre-existing intents whose image is no longer cached.
heartbeat(id, { runtimeImages: [runtime] });
tests/unit/native-creation-intents.test.ts
+5 -4
View File
@@ -55,12 +55,12 @@ describe('native creation intents before placement', () => {
it('cancels unplaced work locally once, retains keys, and frees its logical server name', async () => {
const id = await intent();
const { db } = await import('../../src/lib/server/db/index.js');
const { saveServerKey, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { reserveServerKey, getServerKey, serverKeyPool } = await import('../../src/lib/server/fleet/server-keys.js');
const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
const { nativeDeployment, unplacedNativeCreation } = await import('../../src/lib/server/backends/native.js');
const { nextLobbyNumber } = await import('../../src/lib/server/provisioning/instances.js');
saveServerKey(server, 'saved-key');
reserveServerKey(server, 't5', 'saved-key');
expect(canCancelCreation(id)).toBe(true);
const cleanup = cancelCreation(id)!;
expect(cleanup).toEqual(expect.any(String));
@@ -71,7 +71,8 @@ describe('native creation intents before placement', () => {
expect(getDeployment(cleanup)).toMatchObject({ backend: 'native', deployment: { state: 'done', server_deleted_at: expect.any(String) } });
expect(unplacedNativeCreation(server)).toBeNull();
expect(nextLobbyNumber('t5_zombie_theater')).toBe(1);
expect(getServerKey(server)).toBe('saved-key');
expect(getServerKey(server)).toBeUndefined();
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual([]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
});
@@ -142,7 +143,7 @@ describe('native creation intents before placement', () => {
const { queueNativeCreation, processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', libraryId: library.id, image, plutoKey: 'saved-key' };
const blocked = queueNativeCreation({ ...input, hostId: offline.id });
const available = queueNativeCreation({ ...input, hostId: ready.id });
const available = queueNativeCreation({ ...input, hostId: ready.id, plutoKey: 'ready-key' });
expect(blocked.ok && available.ok).toBe(true);
if (!blocked.ok || !available.ok) throw new Error('Could not queue launch fixtures.');
await processNativeProvisioning(() => null);
tests/unit/native-launch-preparation.test.ts
+2 -1
View File
@@ -43,10 +43,11 @@ afterEach(async () => {
vi.unstubAllEnvs();
});
let queuedKeys = 0;
async function queue() {
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const result = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater',
hostId, libraryId, image, plutoKey: 'saved-key' });
hostId, libraryId, image, plutoKey: `saved-key-${++queuedKeys}` });
if (!result.ok) throw new Error(result.message);
return result;
}
tests/unit/native-provisioning.test.ts
+94 -14
View File
@@ -12,11 +12,12 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) }));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
let queuedKeys = 0;
let fixture: Fixture; let hostId: string; let libraryId: string;
const image = 'example.test/plutainer:test';
const inventory = () => ({ cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] });
beforeEach(async () => {
vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = '';
vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = ''; queuedKeys = 0;
fixture = await newFixture([]);
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password');
@@ -48,7 +49,7 @@ async function queue(game: 't5' | 't6' = 't5', placement: 'pinned' | 'automatic'
await prepareLibrary(game); remote.map = game === 't5' ? 'zombie_theater' : 'zm_transit';
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
const result = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town',
hostId: placement === 'pinned' ? hostId : undefined, libraryId, image, plutoKey: 'saved-key' });
hostId: placement === 'pinned' ? hostId : undefined, libraryId, image, plutoKey: ++queuedKeys === 1 ? 'saved-key' : `saved-key-${queuedKeys}` });
if (!result.ok) throw new Error(result.message);
return result;
}
@@ -990,8 +991,9 @@ describe('native instance provisioning', () => {
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind LIKE 'instance.%'").get()).toEqual({ count: 0 });
const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js');
expect(getLibraryVersion(libraryId)?.state).toBe('ready');
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
expect(getServerKey(job.name)).toBe('saved-key');
const { getServerKey, serverKeyPool } = await import('../../src/lib/server/fleet/server-keys.js');
expect(getServerKey(job.name)).toBeUndefined();
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
});
});
@@ -1083,7 +1085,7 @@ describe('automatic capacity admission and placement', () => {
const preview = previewCollection(blueprint.id);
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 });
const result = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) });
const result = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) });
expect(result).toMatchObject({ ok: true, collectionId: expect.any(String) });
await advance();
const { db } = await import('../../src/lib/server/db/index.js');
@@ -1117,7 +1119,7 @@ describe('automatic capacity admission and placement', () => {
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const measured = { ...inventory(), cpuCount: 1, cpuUsagePercent: 50 };
heartbeat(hostId, measured);
const next = queueNativeCreation(input);
const next = queueNativeCreation({ ...input, plutoKey: 'next-key' });
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
await advance();
expect(await operationFor(next.deploymentId)).toBeNull();
@@ -1151,7 +1153,7 @@ describe('automatic capacity admission and placement', () => {
const blueprint = saveBlueprint({ name: 'Staggered lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 3 }] });
const preview = previewCollection(blueprint.id);
expect(preview.ready).toBe(true);
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) }).ok).toBe(true);
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }).ok).toBe(true);
await advance();
const creates = () => db().prepare("SELECT instance_id FROM ec_host_operations WHERE kind='instance.create'").all() as { instance_id: string }[];
expect(creates()).toHaveLength(1);
@@ -1170,7 +1172,7 @@ describe('automatic capacity admission and placement', () => {
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
const first = queueNativeCreation(input);
expect(first.ok).toBe(true); if (!first.ok) return;
const second = queueNativeCreation(input);
const second = queueNativeCreation({ ...input, plutoKey: 'second-key' });
expect(second).toMatchObject({ ok: true, name: 't5_zombie_theater_2', port: null });
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
@@ -1191,7 +1193,7 @@ describe('automatic capacity admission and placement', () => {
(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,4,?,?,?)`)
.run(hostId, new Date(Date.now() - minute * 60_000).toISOString(), resource === 'cpu' ? 100 : 10,
4 * GiB, resource === 'memory' ? GiB / 8 : Math.floor(3.8 * GiB));
const jobs = Array.from({ length: 3 }, () => queueNativeCreation(input));
const jobs = Array.from({ length: 3 }, (_, index) => queueNativeCreation({ ...input, plutoKey: `key-${index}` }));
expect(jobs.every(job => job.ok)).toBe(true);
await advance();
expect(await operationFor(jobs[0].ok ? jobs[0].deploymentId : 'missing')).not.toBeNull();
@@ -1205,7 +1207,7 @@ describe('automatic capacity admission and placement', () => {
const { db } = await import('../../src/lib/server/db/index.js');
const GiB = 1024 ** 3;
heartbeat(hostId, { ...inventory(), cpuCount: 4, memoryTotal: 4 * GiB, memoryAvailable: Math.floor(3.8 * GiB) });
const jobs = Array.from({ length: 3 }, () => queueNativeCreation(input));
const jobs = Array.from({ length: 3 }, (_, index) => queueNativeCreation({ ...input, plutoKey: `key-${index}` }));
expect(jobs.every(job => job.ok)).toBe(true);
// Learning increases the startup allowance after admission. All three
// no longer fit concurrently, but this otherwise empty host fits one.
@@ -1244,7 +1246,7 @@ describe('automatic capacity admission and placement', () => {
// Adding a new 1 GiB server fits below 2.53 GiB usable memory.
heartbeat(hostId, { ...host, memoryAvailable: 2 * GiB, instanceResources: [{ instanceId: instance.id,
generation: instance.generation, uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
const second = queueNativeCreation(input);
const second = queueNativeCreation({ ...input, plutoKey: 'second-key' });
expect(second).toMatchObject({ ok: true });
await advance();
expect(await operationFor(second.ok ? second.deploymentId : 'missing')).not.toBeNull();
@@ -1265,7 +1267,7 @@ describe('automatic capacity admission and placement', () => {
heartbeat(hostId, { ...inventory(), memoryTotal: 3 * GiB, memoryAvailable: 2 * GiB,
instanceResources: evidence === 'missing' ? [] : [{ instanceId: instance.id, generation: instance.generation + 1,
uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
const next = queueNativeCreation(input);
const next = queueNativeCreation({ ...input, plutoKey: 'next-key' });
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
await advance();
expect(await operationFor(next.deploymentId)).toBeNull();
@@ -1318,7 +1320,7 @@ describe('automatic capacity admission and placement', () => {
await advance();
expect(nativeInstance(result.name)?.host_id).toBe(other.id);
updateHost(other.id, { mode: 'draining' });
const next = queueNativeCreation(input);
const next = queueNativeCreation({ ...input, plutoKey: 'next-key' });
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
await advance();
expect(await operationFor(next.deploymentId)).toBeNull();
@@ -1333,7 +1335,7 @@ describe('automatic capacity admission and placement', () => {
const preview = previewCollection(blueprint.id);
expect(preview.ready).toBe(true);
expect(preview.servers.map(server => server.port)).toEqual([null, null]);
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) }).ok).toBe(true);
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }).ok).toBe(true);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
@@ -1481,6 +1483,7 @@ describe('cancelled native mutation bookkeeping', () => {
it.each(['rebuild','destroy'] as const)('reconciles a delivered %s receipt without resuming the cancelled job', async action => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { db } = await import('../../src/lib/server/db/index.js');
const { serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
@@ -1491,6 +1494,7 @@ describe('cancelled native mutation bookkeeping', () => {
await advance(); const operation = (await operationFor(id))!;
pendingHostOperations(hostId);
expect(cancelProvisioningJob(id)).toBe(true);
expect(serverKeyPool('t5').available).toBe(0);
const replacement = action === 'rebuild' ? requestLifecycle(job.name, 'restart') : null;
if (replacement && !replacement.ok) throw new Error(replacement.error);
await advance();
@@ -1498,6 +1502,8 @@ describe('cancelled native mutation bookkeeping', () => {
await advance(); await advance();
if (replacement?.ok) expect(await operationFor(replacement.deploymentId)).toMatchObject({ kind: 'instance.restart', generation: 2 });
expect((await detail(id)).state).toBe('cancelled');
expect(serverKeyPool('t5').available).toBe(action === 'destroy' ? 1 : 0);
expect(getServerKey(job.name)).toBe(action === 'destroy' ? undefined : 'saved-key');
if(action === 'rebuild') expect(db().prepare('SELECT generation,retired_at FROM native_instances WHERE server=?').get(job.name)).toEqual({generation:2,retired_at:null});
else {
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
@@ -1508,6 +1514,7 @@ describe('cancelled native mutation bookkeeping', () => {
it.each([false,true])('cancels a host migration without restarting it or leaking destination reservations (delivered=%s)', async delivered => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const hosts = await import('../../src/lib/server/hosts/index.js');
const { serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
@@ -1521,12 +1528,15 @@ describe('cancelled native mutation bookkeeping', () => {
expect(migration.remove_operation_id).toBeTruthy();
if(delivered) hosts.pendingHostOperations(hostId);
expect(cancelProvisioningJob(migration.deployment_id)).toBe(true);
expect(serverKeyPool('t5').available).toBe(0);
if(delivered) hosts.reportHostOperation(hostId,migration.remove_operation_id,{state:'completed'});
await processProvisioning(); await processProvisioning();
expect((await detail(migration.deployment_id)).state).toBe('cancelled');
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(destination.id)).toEqual([]);
expect(db().prepare('SELECT * FROM ec_host_migrations').all()).toHaveLength(1);
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toHaveLength(delivered ? 0 : 2);
expect(serverKeyPool('t5').available).toBe(delivered ? 1 : 0);
expect(getServerKey(job.name)).toBe(delivered ? undefined : 'saved-key');
});
});
@@ -1547,3 +1557,73 @@ describe('additional concurrency admission', () => {
expect(await operationFor(second.deploymentId)).not.toBeNull();
});
});
describe('game key pool launch lifecycle', () => {
it('reserves across maps atomically, rejects duplicate credentials, and reuses a cancelled launch key', async () => {
await prepareLibrary();
const { saveServerKey, serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { queueCreateServer, destroyServer, previewCreation } = await import('../../src/lib/server/provisioning/instances.js');
saveServerKey('t5', 'shared-t5-key', 'Any T5 map');
saveServerKey('t6', 'shared-t6-key');
const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'moon', hostId, libraryId, image };
expect(previewCreation(input).hasSavedKey).toBe(true);
const first = queueCreateServer(input);
if (!first.ok) throw new Error(first.message);
expect(first.name).toBe('t5_zombie_moon_1');
expect(getServerKey(first.name)).toBe('shared-t5-key');
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 0 });
expect(serverKeyPool('t6')).toEqual({ total: 1, available: 1 });
expect(previewCreation({ ...input, mapSlug: 'theater' }).hasSavedKey).toBe(false);
expect(queueCreateServer({ ...input, mapSlug: 'theater' }).ok).toBe(false);
expect(queueCreateServer({ ...input, mapSlug: 'theater', plutoKey: 'shared-t5-key' }).ok).toBe(false);
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
expect(serverKeyPool('t5').available).toBe(0);
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1);
expect(destroyServer(first.name).ok).toBe(true);
expect(serverKeyPool('t5').available).toBe(1);
const second = queueCreateServer({ ...input, mapSlug: 'theater' });
if (!second.ok) throw new Error(second.message);
expect(getServerKey(second.name)).toBe('shared-t5-key');
expect(getServerKey(first.name)).toBeUndefined();
});
it('keeps a key through stop and rebuild, then releases it only after confirmed removal', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
const stopped = requestLifecycle(job.name, 'stop');
if (!stopped.ok) throw new Error(stopped.error);
await advance(); remote.state = 'stopped'; await acknowledge(stopped.deploymentId);
expect(serverKeyPool('t5').available).toBe(0);
const rebuilt = requestLifecycle(job.name, 'rebuild');
if (!rebuilt.ok) throw new Error(rebuilt.error);
await advance(); remote.state = 'running'; await acknowledge(rebuilt.deploymentId);
expect(getServerKey(job.name)).toBe('saved-key');
expect(serverKeyPool('t5').available).toBe(0);
const removal = destroyServer(job.name);
if (!removal.ok || !removal.deploymentId) throw new Error('Expected removal');
await advance();
expect(serverKeyPool('t5').available).toBe(0);
const operation = (await operationFor(removal.deploymentId))!;
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
reportHostOperation(hostId, operation.id, { state: 'unknown', error: 'Receipt lost' });
await advance();
expect(serverKeyPool('t5').available).toBe(0);
await acknowledge(removal.deploymentId);
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
expect(getServerKey(job.name)).toBeUndefined();
});
it('keeps the reservation for a submitted cancellation until the host confirms cleanup', async () => {
const job = await queue(); await advance();
const { serverKeyPool } = await import('../../src/lib/server/fleet/server-keys.js');
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
const cleanup = cancelCreation(job.deploymentId)!;
expect(cleanup).toEqual(expect.any(String));
expect(serverKeyPool('t5').available).toBe(0);
await acknowledge(cleanup);
expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
});
});
tests/unit/native-registry-migration.test.ts
+3
View File
@@ -12,6 +12,9 @@ it('retires only obsolete backend columns while preserving instances, history, k
const previous = new Database(fixture.dbPath);
// Model the former required resource identity and an unrelated extension.
previous.exec(schema);
previous.exec(`DROP TABLE server_keys;
CREATE TABLE server_keys(server_id TEXT PRIMARY KEY,pluto_key TEXT NOT NULL,
updated_at TEXT NOT NULL DEFAULT (datetime('now')));`);
previous.exec(`ALTER TABLE servers ADD COLUMN komodo_type TEXT NOT NULL DEFAULT 'deployment';
ALTER TABLE servers ADD COLUMN komodo_name TEXT NOT NULL;
ALTER TABLE servers ADD COLUMN komodo_server TEXT;
tests/unit/runtime-image-policy.test.ts
+1 -1
View File
@@ -81,7 +81,7 @@ describe('runtime image launch policy', () => {
expect(() => previewCollection(blueprint.id, 'fill', { image: custom })).toThrow(/prepared by an admin/);
await cache([custom]);
const preview = previewCollection(blueprint.id, 'fill', { image: custom });
const keys = Object.fromEntries(preview.servers.map(server => [server.name!, 'saved-key']));
const keys = Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name]));
await cache([]);
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(/prepared by an admin/);
const { db } = await import('../../src/lib/server/db/index.js');
tests/unit/server-key-migration.test.ts
+108
View File
@@ -0,0 +1,108 @@
import Database from 'better-sqlite3';
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import schema from '../../src/lib/server/db/schema.sql?raw';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { await fixture.cleanup(); });
it('migrates idle, occupied and duplicate legacy keys without releasing unresolved assignments', async () => {
const previous = new Database(fixture.dbPath);
previous.exec(schema);
previous.exec(`DROP TABLE server_keys;
CREATE TABLE server_keys(server_id TEXT PRIMARY KEY,pluto_key TEXT NOT NULL,updated_at TEXT NOT NULL DEFAULT (datetime('now')));`);
const { migrateNativeInstances } = await import('../../src/lib/server/backends/schema.js');
migrateNativeInstances(previous);
const cases = [
['t5_zombie_theater_1', 'shared-occupied'],
['t5_zombie_pentagon_1', 'shared-occupied'],
['t5_zombie_cosmodrome_1', 'native-only'],
['t5_zombie_temple_1', 'queued-key'],
['t5_zombie_sumpf_1', 'failed-key'],
['t5_zombie_moon_1', 'idle-key'],
['t5_zombie_moon_2', 'idle-key'],
['t6_zombie_town_1', 't6-key']
];
for (const [server, credential] of cases) previous.prepare('INSERT INTO server_keys(server_id,pluto_key,updated_at) VALUES(?,?,?)').run(server, credential, '2026-01-01');
previous.exec(`INSERT INTO servers(name,game,desired_state) VALUES('t5_zombie_theater_1','t5','running'),('t5_zombie_pentagon_1','t5','stopped');
INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
VALUES('native','t5_zombie_cosmodrome_1','host','library','image',28960,'2026-01-01');
INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) VALUES
('queued','t5_zombie_temple_1','create','queued','Queued','2026-01-01','fixture'),
('failed','t5_zombie_sumpf_1','create','failed','Placement failed','2026-01-01','fixture');`);
previous.close();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const pool = await import('../../src/lib/server/fleet/server-keys.js');
for (let pass = 0; pass < 2; pass++) {
expect(pool.serverKeyPool('t5')).toEqual({ total: 5, available: 1 });
expect(pool.serverKeyPool('t6')).toEqual({ total: 1, available: 1 });
expect(pool.getServerKey('t5_zombie_theater_1')).toBe('shared-occupied');
expect(pool.getServerKey('t5_zombie_pentagon_1')).toBe('shared-occupied');
expect(pool.getServerKey('t5_zombie_cosmodrome_1')).toBe('native-only');
expect(pool.getServerKey('t5_zombie_temple_1')).toBe('queued-key');
expect(pool.getServerKey('t5_zombie_sumpf_1')).toBe('failed-key');
expect(pool.getServerKey('t5_zombie_moon_1')).toBeUndefined();
expect(db().pragma('foreign_key_check')).toEqual([]);
closeDb();
}
const duplicate = pool.listServerKeys().find(key => key.assigned_servers.length === 2)!;
expect(duplicate.assigned_servers).toEqual(['t5_zombie_pentagon_1', 't5_zombie_theater_1']);
expect(duplicate.updated_at).toBe('2026-01-01');
expect(pool.reserveServerKey('t5_zombie_theater_2', 't5')).toBe('idle-key');
pool.releaseServerKey('t5_zombie_theater_1');
expect(pool.availableServerKeyCount('t5')).toBe(0);
expect(() => pool.removeServerKey(duplicate.id)).toThrow('reserved');
pool.releaseServerKey('t5_zombie_pentagon_1');
expect(pool.reserveServerKey('t5_zombie_theater_3', 't5')).toBe('shared-occupied');
});
it('recovers actual launch credentials and holds both sides of an unresolved legacy rebuild', async () => {
const previous = new Database(fixture.dbPath);
previous.exec(schema);
previous.exec(`DROP TABLE server_keys;
CREATE TABLE server_keys(server_id TEXT PRIMARY KEY,pluto_key TEXT NOT NULL,updated_at TEXT NOT NULL DEFAULT (datetime('now')));`);
const { migrateNativeInstances } = await import('../../src/lib/server/backends/schema.js');
const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js');
migrateNativeInstances(previous);
ensureHostSchema(previous);
previous.exec(`INSERT INTO ec_hosts(id,name,created_at) VALUES('host','Fixture host','2026-01-01');
INSERT INTO servers(name,game) VALUES('t5_zombie_theater_1','t5'),('t5_zombie_pentagon_1','t5');
INSERT INTO server_keys(server_id,pluto_key) VALUES('t5_zombie_theater_1','edited-saved-key'),
('t5_zombie_pentagon_1','original-key'),('t5_zombie_moon_1','original-key');
INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES
('kino','t5_zombie_theater_1','host',2,'library','image',28960,'2026-01-01'),
('five','t5_zombie_pentagon_1','host',1,'library','image',28961,'2026-01-01'),
('queued','t5_zombie_cosmodrome_1','host',1,'library','image',28962,'2026-01-01');`);
function operation(id: string, instance: string, generation: number, kind: string, state: string, key: string, staged = false, nextGeneration?: number) {
const payload = JSON.stringify({ game: 't5', env: { PLUTO_SERVER_KEY: key }, nextGeneration });
previous.prepare(`INSERT INTO ec_host_operations(id,host_id,instance_id,generation,kind,state,payload,created_at,updated_at)
VALUES(?,'host',?,?,?,?,?,'2026-01-01','2026-01-01')`).run(id, instance, generation, kind, state, staged ? '{}' : payload);
if (staged) previous.prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,?)').run(id, payload);
}
operation('obsolete-kino', 'kino', 1, 'instance.create', 'completed', 'obsolete-key');
operation('actual-kino', 'kino', 1, 'instance.rebuild', 'completed', 'actual-key', true, 2);
operation('original-five', 'five', 1, 'instance.create', 'completed', 'original-key');
operation('unknown-five', 'five', 1, 'instance.rebuild', 'unknown', 'pending-key', true, 2);
operation('unknown-creation', 'queued', 1, 'instance.create', 'unknown', 'unknown-creation-key');
operation('failed-new-key', 'kino', 2, 'instance.rebuild', 'failed', 'never-applied-key', true, 3);
previous.close();
const pool = await import('../../src/lib/server/fleet/server-keys.js');
expect(pool.getServerKey('t5_zombie_theater_1')).toBe('actual-key');
expect(pool.getServerKey('t5_zombie_pentagon_1')).toBe('pending-key');
expect(pool.getServerKey('t5_zombie_cosmodrome_1')).toBe('unknown-creation-key');
expect(pool.serverKeyPool('t5')).toEqual({ total: 5, available: 1 });
expect(pool.serverKeyPool('t5', 't5_zombie_pentagon_1')).toEqual({ total: 5, available: 2 });
expect(() => pool.reserveServerKey('t5_zombie_moon_2', 't5', 'original-key')).toThrow('already reserved');
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
expect(db().prepare(`SELECT k.pluto_key,l.is_current FROM server_key_leases l JOIN server_keys k ON k.id=l.key_id
WHERE l.server_id='t5_zombie_pentagon_1' ORDER BY l.is_current`).all())
.toEqual([{ pluto_key: 'original-key', is_current: 0 }, { pluto_key: 'pending-key', is_current: 1 }]);
expect(db().pragma('foreign_key_check')).toEqual([]);
closeDb();
expect(pool.serverKeyPool('t5')).toEqual({ total: 5, available: 1 });
pool.releaseServerKey('t5_zombie_pentagon_1');
expect(pool.availableServerKeyCount('t5')).toBe(3);
});
tests/unit/server-key-pool.test.ts
+92
View File
@@ -0,0 +1,92 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { await fixture.cleanup(); });
const kino = 't5_zombie_theater_1';
const five = 't5_zombie_pentagon_1';
const town = 't6_zombie_town_1';
async function keys() { return import('../../src/lib/server/fleet/server-keys.js'); }
describe('per-game key pools', () => {
it('shares keys across maps only after their previous reservation is released', async () => {
const pool = await keys();
const id = pool.saveServerKey('t5', 'secret-t5', 'Shared T5 key');
pool.saveServerKey('t6', 'secret-t6');
expect(pool.reserveServerKey(kino, 't5')).toBe('secret-t5');
expect(pool.reserveServerKey(kino, 't5')).toBe('secret-t5');
expect(pool.serverKeyPool('t5')).toEqual({ total: 1, available: 0 });
expect(pool.serverKeyPool('t5', kino)).toEqual({ total: 1, available: 1 });
expect(() => pool.reserveServerKey(five, 't5')).toThrow('No available T5 server keys');
expect(() => pool.removeServerKey(id)).toThrow('reserved');
expect(pool.reserveServerKey(town, 't6')).toBe('secret-t6');
expect(pool.getServerKey(five)).toBeUndefined();
expect(pool.releaseServerKey(kino)).toBe(true);
expect(pool.reserveServerKey(five, 't5')).toBe('secret-t5');
expect(pool.listServerKeys().find(key => key.id === id)).toMatchObject({ label: 'Shared T5 key', assigned_servers: [five] });
pool.releaseServerKey(five);
expect(pool.removeServerKey(id)).toBe(true);
expect(pool.removeServerKey(id)).toBe(false);
});
it('deduplicates credentials and exposes only metadata', async () => {
const pool = await keys();
const id = pool.saveServerKey('t5', ' secret ', ' First ');
expect(pool.saveServerKey('t5', 'secret', 'Renamed')).toBe(id);
expect(pool.saveServerKey('t5', 'secret')).toBe(id);
expect(pool.serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
expect(pool.listServerKeys()).toEqual([{ id, game: 't5', label: 'Renamed', updated_at: expect.any(String), assigned_servers: [] }]);
expect(JSON.stringify(pool.listServerKeys())).not.toContain('secret');
for (const [game, credential, label] of [['t7', 'valid', ''], ['t5', 'two words', ''], ['t5', '', ''], ['t5', 'valid', 'x'.repeat(161)]]) {
expect(() => pool.saveServerKey(game, credential, label)).toThrow();
}
});
it('reserves supplied keys atomically and never substitutes another available key', async () => {
const pool = await keys();
expect(pool.reserveServerKey(kino, 't5', 'manual-key')).toBe('manual-key');
pool.saveServerKey('t5', 'spare-key');
expect(() => pool.reserveServerKey(five, 't5', 'manual-key')).toThrow('already reserved');
expect(pool.getServerKey(five)).toBeUndefined();
expect(pool.availableServerKeyCount('t5')).toBe(1);
expect(() => pool.reserveServerKey(kino, 't5', 'replacement')).toThrow('already has a reserved key');
expect(() => pool.reserveServerKey(kino, 't6')).toThrow('another game');
expect(pool.reserveServerKey(five, 't5', '')).toBe('spare-key');
});
it('rolls back reservation and imported credentials when launch admission fails', async () => {
const pool = await keys();
const { db } = await import('../../src/lib/server/db/index.js');
expect(() => db().transaction(() => {
pool.reserveServerKey(kino, 't5', 'temporary-key');
throw new Error('Launch validation failed');
})()).toThrow('Launch validation failed');
expect(pool.listServerKeys()).toEqual([]);
expect(pool.getServerKey(kino)).toBeUndefined();
});
it('keeps reservations across restart and missing registry rows', async () => {
const pool = await keys();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
pool.reserveServerKey(kino, 't5', 'retained-key');
db().prepare("INSERT INTO servers(name,game,desired_state) VALUES(?,'t5','stopped')").run(kino);
closeDb();
expect(pool.getServerKey(kino)).toBe('retained-key');
expect(pool.availableServerKeyCount('t5')).toBe(0);
db().prepare('DELETE FROM servers WHERE name=?').run(kino);
expect(pool.availableServerKeyCount('t5')).toBe(0);
});
it('does not allocate a duplicated credential imported into another game', async () => {
const pool = await keys();
pool.reserveServerKey(kino, 't5', 'same-secret');
const duplicate = pool.saveServerKey('t6', 'same-secret');
expect(pool.serverKeyPool('t6')).toEqual({ total: 1, available: 0 });
expect(pool.listServerKeys().find(key => key.id === duplicate)?.assigned_servers).toEqual([kino]);
expect(() => pool.removeServerKey(duplicate)).toThrow('reserved');
expect(() => pool.reserveServerKey(town, 't6')).toThrow('No available T6');
});
});
tests/unit/server-keys-api.test.ts
+70
View File
@@ -0,0 +1,70 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
function request(method: string, body: unknown) {
return { request: new Request('http://localhost/api/server-keys', {
method, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body)
}) };
}
describe('game key pool API', () => {
it('returns pool metadata and assigned servers without credentials', async () => {
const route = await import('../../src/routes/api/server-keys/+server.js');
const { reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const saved = await route.PUT(request('PUT', { game: 't5', label: 'Cherry hostname 5', plutoKey: 'private-t5-credential' }));
expect(saved.status).toBe(200);
const { id } = await saved.json();
reserveServerKey('t5_zombie_theater_2', 't5');
const response = route.GET();
expect(response.headers.get('cache-control')).toBe('no-store');
const text = await response.text();
expect(text).not.toContain('private-t5-credential');
expect(text).not.toContain('pluto_key');
expect(text).not.toContain('plutoKey');
expect(JSON.parse(text)).toEqual({ keys: [{
id, game: 't5', label: 'Cherry hostname 5', assigned_servers: ['t5_zombie_theater_2'], updated_at: expect.any(String)
}] });
});
it.each([
{ game: 't4', plutoKey: 'private-key' },
{ serverId: 't5_zombie_theater_2', plutoKey: 'private-key' },
{ game: 't5', plutoKey: 'contains whitespace' },
{ game: 't5', plutoKey: 'private-key', label: 'x'.repeat(161) }
])('rejects invalid pool input without saving it: %j', async body => {
const route = await import('../../src/routes/api/server-keys/+server.js');
const response = await route.PUT(request('PUT', body));
expect(response.status).toBe(400);
expect(await response.json()).toMatchObject({ ok: false, error: expect.any(String) });
expect(await route.GET().json()).toEqual({ keys: [] });
});
it('deduplicates a credential in its game pool', async () => {
const route = await import('../../src/routes/api/server-keys/+server.js');
const first = await route.PUT(request('PUT', { game: 't6', plutoKey: 'private-key', label: 'First label' }));
const repeated = await route.PUT(request('PUT', { game: 't6', plutoKey: ' private-key ', label: 'Updated label' }));
expect(await repeated.json()).toEqual(await first.json());
expect((await route.GET().json()).keys).toEqual([expect.objectContaining({
game: 't6', label: 'Updated label', assigned_servers: []
})]);
});
it('rejects removal while assigned and permits it after the key is released', async () => {
const route = await import('../../src/routes/api/server-keys/+server.js');
const { reserveServerKey, releaseServerKey, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const saved = await route.PUT(request('PUT', { game: 't5', plutoKey: 'private-key' }));
const { id } = await saved.json();
reserveServerKey('t5_zombie_moon_1', 't5');
const blocked = await route.DELETE(request('DELETE', { id }));
expect(blocked.status).toBe(400);
expect(await blocked.json()).toMatchObject({ ok: false, error: expect.stringMatching(/reserved|assigned|in use/i) });
expect(getServerKey('t5_zombie_moon_1')).toBe('private-key');
releaseServerKey('t5_zombie_moon_1');
expect(await (await route.DELETE(request('DELETE', { id }))).json()).toEqual({ ok: true });
expect(await route.GET().json()).toEqual({ keys: [] });
});
});
tests/unit/watchdog.test.ts
+4 -4
View File
@@ -31,8 +31,8 @@ beforeEach(async () => {
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
await seedServerRegistry();
database.db().prepare('UPDATE servers SET game_port = 28960 WHERE name = ?').run(server);
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
saveServerKey(server, 'saved-key');
const { reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
reserveServerKey(server, 't5', 'saved-key');
refreshHost = (await prepareNativeHost([server])).heartbeat;
backend = await import('../../src/lib/server/backends/native.js');
vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' });
@@ -504,8 +504,8 @@ describe('Watchdog APIs', () => {
describe('Watchdog drift campaigns', () => {
it('keeps a missing rebuild key visible as attention without deleting files or retrying automatically', async () => {
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
removeServerKey(server);
// Simulate legacy registry damage; administrators cannot remove a leased key.
database.db().prepare('DELETE FROM server_key_leases WHERE server_id=?').run(server);
const path = join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc');
await writeFile(path, 'unexpected');
await watchdog.watchdogTick(true);