From 4dbf704e6fffa7cb183a7a000ed14e5f4d843e9e Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Tue, 6 Oct 2026 23:37:20 -0400 Subject: [PATCH] Require full key coverage when saving autoscaling policies --- docs/autoscaling.md | 2 +- docs/lobby-autoscaling-plan.md | 4 ++ src/lib/server/autoscaling/index.ts | 10 +++- src/lib/server/autoscaling/snapshot.ts | 3 +- src/routes/autoscaling/+page.svelte | 1 + tests/unit/autoscaling-dry-run.test.ts | 9 ++- tests/unit/autoscaling-keys.test.ts | 82 ++++++++++++++++++++++++++ 7 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 tests/unit/autoscaling-keys.test.ts diff --git a/docs/autoscaling.md b/docs/autoscaling.md index ad8df39..d1e140e 100644 --- a/docs/autoscaling.md +++ b/docs/autoscaling.md @@ -17,7 +17,7 @@ Choose **Preview changes** to inspect the existing lobbies, saved key coverage, | Create only | Adds a lobby when eligible. Surplus removal remains advisory. | | Create and destroy | Adds lobbies and removes surplus lobbies after continuous empty evidence and all removal checks pass. | -Live activation requires saved Plutonium keys for enough exact lobby identities to support the configured maximum. Keys may be nonconsecutive: if only lobby 3 and lobby 8 are keyed, those identities can be used. Existing lobbies and unfinished operations must be healthy and settled before activation. Automatic removal also requires agent **1.7.0 or later** advertising `instance.autoscaling-remove` on each affected host. +Creating a policy in any mode, including Dry run or Off, requires saved Plutonium keys for enough exact lobby identities to support the configured maximum on every selected map. Preview lists each shortage, and Save rechecks coverage transactionally. Non-Off edits also require coverage; an existing policy can always be turned Off if keys have been removed. Only supported lobby identities numbered 1–20 with nonblank saved keys count. Keys may be nonconsecutive: if only lobby 3 and lobby 8 are keyed, those identities can be used. Existing lobbies and unfinished operations must be healthy and settled before activation. Automatic removal also requires agent **1.7.0 or later** advertising `instance.autoscaling-remove` on each affected host. The desired count is `min(maximum, max(minimum, occupied lobbies + empty spare target))`. The controller evaluates policies every 15 seconds and requests at most one unresolved operation per policy. Game-wide policies share one cooldown and one outstanding operation across their maps, prioritizing additions before removals. Create uses the existing startup concurrency, CPU, memory, library, port, and host admission checks. Lack of host capacity leaves one request waiting; it does not create additional requests every tick. Creates use current promoted launch files when provisioning begins. diff --git a/docs/lobby-autoscaling-plan.md b/docs/lobby-autoscaling-plan.md index 3bf607c..8e673be 100644 --- a/docs/lobby-autoscaling-plan.md +++ b/docs/lobby-autoscaling-plan.md @@ -148,3 +148,7 @@ The full controller suite reported 1,934 passing tests and two existing failures An isolated Firefox review exercised policy creation, preview, save, keyboard focus, and 1440, 390, and 320 pixel layouts. A narrow-layout heading issue found during review was corrected. The preview used a temporary database and no production host credentials. Production CPU savings, external joinability, spare targets, and acceptable deficit durations still require the two-week observation period and operator-run pilot. Game-wide policy follow-up: selecting every map applies the targets independently to each catalog map and variant. Game-wide policies serialize actions across maps and cannot overlap live map-specific ownership. Existing map policies remain compatible. + +Key coverage follow-up: creation in every mode requires sufficient saved keys for the maximum on each selected map. Coverage is rechecked on save, including edits to active or dry-run policies. Turning an existing policy Off remains available after key loss. + +Follow-up validation: 40 autoscaling tests and 118 related provisioning, collection, and cancellation tests passed, along with type checking and the production build. An isolated Firefox review verified that missing per-map keys disable Save, complete coverage permits game-wide dry-run creation, and keyboard navigation and 1440, 390, and 320 pixel layouts work. diff --git a/src/lib/server/autoscaling/index.ts b/src/lib/server/autoscaling/index.ts index 6ec369a..4840b8e 100644 --- a/src/lib/server/autoscaling/index.ts +++ b/src/lib/server/autoscaling/index.ts @@ -30,12 +30,16 @@ export function previewScaling(input: unknown, id?: string, revision?: number): nextActionAt: existing?.nextActionAt ?? 0, pausedReason: null, updatedAt: existing?.updatedAt ?? '' }; const snapshot = scalingSnapshot(policy); const blockers: string[] = []; + // Creating any policy requires full coverage; existing policies must always be pausable. + if (!existing || policy.mode !== 'off') { + for (const map of zombieMaps(policy.game).filter(m => policy.mapSlug === '*' || m.slug === policy.mapSlug)) { + const count = snapshot.keyedNames.filter(name => parseServerName(name)?.mapSlug === map.slug).length; + if (count < policy.maximum) blockers.push(`Save keys for at least ${policy.maximum} lobby identities for ${map.friendly} before saving this policy (${count} saved; ${policy.maximum - count} more required).`); + } + } if (liveScaling(policy.mode)) { const owner = scalingOwner(policy.game, policy.mapSlug); if (owner && owner !== policy.id) blockers.push(`An overlapping policy owns this lineup. Pause it and settle pending work first: /autoscaling#${owner}`); - for (const map of zombieMaps(policy.game).filter(m => policy.mapSlug === '*' || m.slug === policy.mapSlug)) { - if (snapshot.keyedNames.filter(name => parseServerName(name)?.mapSlug === map.slug).length < policy.maximum) blockers.push(`Save keys for at least ${policy.maximum} lobby identities for ${map.friendly} before activating this policy.`); - } blockers.push(...snapshot.blockers); for (const member of snapshot.members) if (!['occupied', 'empty'].includes(member.state)) blockers.push(`${member.name}: ${member.blocker ?? member.state}. Settle this work before activation.`); if (policy.mode === 'create-destroy') for (const member of snapshot.members) { diff --git a/src/lib/server/autoscaling/snapshot.ts b/src/lib/server/autoscaling/snapshot.ts index c20b305..cf4929a 100644 --- a/src/lib/server/autoscaling/snapshot.ts +++ b/src/lib/server/autoscaling/snapshot.ts @@ -1,4 +1,5 @@ import { db } from '../db/index.js'; +import { validServerId } from '../fleet/server-keys.js'; import { listServers, type ServerRow } from '../fleet/servers.js'; import { observationReader } from '../fleet/status.js'; import { parseServerName, zombieMaps } from '../../map-aliases.js'; @@ -72,6 +73,6 @@ export function scalingSnapshot(policy: ScalingPolicy, now = Date.now(), exceptO reservedNames.add(row.name); if (!members.some(m => m.name === row.name)) blockers.push(`Unresolved native instance ${row.name}.`); } - const keyedNames = (handle.prepare("SELECT server_id AS name FROM server_keys WHERE length(trim(pluto_key))>0").all() as { name: string }[]).filter(matches).map(r => r.name); + const keyedNames = (handle.prepare("SELECT server_id AS name FROM server_keys WHERE length(trim(pluto_key))>0").all() as { name: string }[]).filter(row => validServerId(row.name) && matches(row)).map(r => r.name); return { members, reservedNames: [...reservedNames], keyedNames, blockers }; } diff --git a/src/routes/autoscaling/+page.svelte b/src/routes/autoscaling/+page.svelte index 8d58c2c..df9838d 100644 --- a/src/routes/autoscaling/+page.svelte +++ b/src/routes/autoscaling/+page.svelte @@ -83,6 +83,7 @@ {#if settings.mapSlug === '*'}

Targets apply separately to every map, including all variants. Each map needs its own keys. Actions share one cooldown and run one at a time across the game.

{/if} +

Saved lobby keys must cover the maximum for each selected map before you can save a new policy, including Dry run.

diff --git a/tests/unit/autoscaling-dry-run.test.ts b/tests/unit/autoscaling-dry-run.test.ts index 167ad18..8d86c78 100644 --- a/tests/unit/autoscaling-dry-run.test.ts +++ b/tests/unit/autoscaling-dry-run.test.ts @@ -3,14 +3,17 @@ import { newFixture, type Fixture } from '../helpers/fixture.js'; import { routePermission, roleAllows } from '../../src/lib/server/auth/access.js'; let fixture: Fixture; -beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +beforeEach(async () => { + vi.resetModules(); fixture = await newFixture([]); + const { db } = await import('../../src/lib/server/db/index.js'); + for (const n of [3, 8, 9, 10]) db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(`t6_zombie_town_${n}`, `secret-${n}`); +}); afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); }); const settings = { game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' }; it('persists deduplicated dry-run decisions across restart without provisioning or reservations', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); const scaling = await import('../../src/lib/server/autoscaling/index.js'); - db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run('t6_zombie_town_3', 'secret'); const preview = scaling.previewScaling(settings); const saved = scaling.saveScaling(settings, undefined, undefined, preview.token); await scaling.runAutoscaling(); await scaling.runAutoscaling(); @@ -31,6 +34,8 @@ it('rejects duplicate policies, stale edits and changed previews', async () => { expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('already exists'); expect(() => scaling.previewScaling(settings, saved.id, 0)).toThrow('changed'); expect(() => scaling.saveScaling({ ...settings, maximum: 5 }, saved.id, saved.revision, preview.token)).toThrow('preview changed'); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_10'").run(); expect(() => scaling.saveScaling({ ...settings, mode: 'create-destroy' }, saved.id, saved.revision, scaling.previewScaling({ ...settings, mode: 'create-destroy' }, saved.id, saved.revision).token)).toThrow('Save keys'); }); diff --git a/tests/unit/autoscaling-keys.test.ts b/tests/unit/autoscaling-keys.test.ts new file mode 100644 index 0000000..2588143 --- /dev/null +++ b/tests/unit/autoscaling-keys.test.ts @@ -0,0 +1,82 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import { prepareNativeHost } from '../helpers/native-host.js'; +import { zombieMaps } from '../../src/lib/map-aliases.js'; + +let fixture: Fixture; +beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +afterEach(async () => { await fixture.cleanup(); }); +const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'dry-run' }; +async function setup() { + const { db } = await import('../../src/lib/server/db/index.js'); + const scaling = await import('../../src/lib/server/autoscaling/index.js'); + const key = (name: string, value = `key-${name}`) => db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(name, value); + return { db, scaling, key }; +} + +it.each(['off', 'dry-run', 'create-only', 'create-destroy'])('rejects new %s policies without enough keys through both service and API', async mode => { + const { db, scaling, key } = await setup(); + key('t6_zombie_town_3'); + const input = { ...settings, mode }; + const preview = scaling.previewScaling(input); + expect(preview.blockers[0]).toContain('1 saved; 1 more required'); + expect(() => scaling.saveScaling(input, undefined, undefined, preview.token)).toThrow('Save keys'); + const { POST } = await import('../../src/routes/api/autoscaling/+server.js'); + const response = await POST({ request: new Request('http://localhost/api/autoscaling', { method: 'POST', body: JSON.stringify({ action: 'save', settings: input, token: preview.token }) }) }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ error: expect.stringContaining('Save keys') }); + expect(db().prepare('SELECT COUNT(*) AS n FROM autoscaling_policies').get()).toEqual({ n: 0 }); +}); + +it('counts only supported, nonblank identities in the selected map and accepts exact nonconsecutive coverage', async () => { + const { scaling, key } = await setup(); + for (const name of ['t6_zombie_town_0', 't6_zombie_town_21', 't6_zombie_town_01', 't6_zombie_town', 't6_town_2', 't5_zombie_theater_1', 't6_zombie_farm_1']) key(name); + key('t6_zombie_town_2', ' '); + key('t6_zombie_town_3'); + expect(scaling.previewScaling(settings).snapshot.keyedNames).toEqual(['t6_zombie_town_3']); + key('t6_zombie_town_8'); + const preview = scaling.previewScaling(settings); + expect(preview.blockers).toEqual([]); + expect(scaling.saveScaling(settings, undefined, undefined, preview.token).maximum).toBe(2); +}); + +it('requires coverage on every game map rather than pooling excess keys from another map', async () => { + const { scaling, key } = await setup(); + for (const map of zombieMaps('t6')) for (const n of [3, 8]) if (map.slug !== 'town' || n === 3) key(`t6_zombie_${map.slug}_${n}`); + key('t6_zombie_farm_9'); + const input = { ...settings, mapSlug: '*' }; + const preview = scaling.previewScaling(input); + expect(preview.blockers).toEqual([expect.stringContaining('Town Survival')]); + expect(() => scaling.saveScaling(input, undefined, undefined, preview.token)).toThrow('Town Survival'); + key('t6_zombie_town_8'); + expect(scaling.saveScaling(input, undefined, undefined, scaling.previewScaling(input).token).mapSlug).toBe('*'); +}); + +it('rechecks key deletion after preview and rejects increases beyond coverage', async () => { + const { db, scaling, key } = await setup(); + key('t6_zombie_town_3'); key('t6_zombie_town_8'); + const preview = scaling.previewScaling(settings); + db().prepare("DELETE FROM server_keys WHERE server_id='t6_zombie_town_8'").run(); + expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('preview changed'); + expect(() => scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token)).toThrow('Save keys'); + key('t6_zombie_town_8'); + const saved = scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token); + const higher = { ...settings, maximum: 3 }; + expect(() => scaling.saveScaling(higher, saved.id, saved.revision, scaling.previewScaling(higher, saved.id, saved.revision).token)).toThrow('Save keys'); + expect(scaling.getScalingPolicy(saved.id)?.maximum).toBe(2); +}); + +it('allows an existing live policy to pause and withdraw pending work after all keys are removed', async () => { + await prepareNativeHost(); + const { db, scaling, key } = await setup(); + key('t6_zombie_town_3'); key('t6_zombie_town_8'); + const live = { ...settings, mode: 'create-only' }; + const saved = scaling.saveScaling(live, undefined, undefined, scaling.previewScaling(live).token); + await scaling.runAutoscaling(); + db().prepare('DELETE FROM server_keys').run(); + const off = { ...settings, mode: 'off' }; + const preview = scaling.previewScaling(off, saved.id, saved.revision); + expect(preview.blockers).toEqual([]); + expect(scaling.saveScaling(off, saved.id, saved.revision, preview.token).mode).toBe('off'); + expect(db().prepare("SELECT state FROM server_deployments WHERE action='create'").get()).toEqual({ state: 'cancelled' }); +});