josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove dry-run and create-only autoscaling modes

Browse Source
This commit is contained in:
josh
2026-10-07 15:49:46 -04:00
parent c9a604303d
commit eadf97a428
18 changed files with 172 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/autoscaling-delete.test.ts
+6 -6
View File
@@ -5,8 +5,8 @@ import { prepareNativeHost } from '../helpers/native-host.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { await fixture.cleanup(); });
const settings = { game: 't6', mapSlug: 'town', maximum: 1, mode: 'dry-run' };
async function setup(mode = 'dry-run') {
const settings = { game: 't6', mapSlug: 'town', maximum: 1, mode: 'create-destroy' };
async function setup(mode = 'create-destroy') {
if (mode.startsWith('create')) await prepareNativeHost();
const { db } = await import('../../src/lib/server/db/index.js');
const scaling = await import('../../src/lib/server/autoscaling/index.js');
@@ -31,10 +31,10 @@ it('deletes policy evidence atomically while preserving lobbies, keys, and unrel
expect(db().prepare('SELECT name FROM servers').all()).toEqual([{ name: 't6_zombie_town_1' }]);
expect(db().prepare('SELECT COUNT(*) AS n FROM server_keys').get()).toEqual({ n: 2 });
expect(db().prepare('PRAGMA foreign_key_check').all()).toEqual([]);
expect(scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token).id).not.toBe(saved.id);
expect(scaling.saveScaling({ ...settings, mode: 'off' }, undefined, undefined, scaling.previewScaling({ ...settings, mode: 'off' }).token).id).not.toBe(saved.id);
});
it.each(['off', 'create-only', 'create-destroy'])('deletes a settled %s policy even after key loss and releases ownership', async mode => {
it.each(['off', 'create-destroy'])('deletes a settled %s policy even after key loss and releases ownership', async mode => {
const { db, scaling, saved } = await setup(mode);
db().prepare('DELETE FROM server_keys').run();
scaling.deleteScaling(saved.id, saved.revision);
@@ -54,11 +54,11 @@ it('rejects stale revisions and preserves the latest settings', async () => {
});
it('blocks deletion with pending intent, then keeps Provisioning history after pause and settlement', async () => {
const { db, scaling, saved } = await setup('create-only');
const { db, scaling, saved } = await setup('create-destroy');
await scaling.runAutoscaling();
const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string };
expect(() => scaling.deleteScaling(saved.id, saved.revision)).toThrow('outstanding Provisioning');
expect(scaling.getScalingPolicy(saved.id)?.mode).toBe('create-only');
expect(scaling.getScalingPolicy(saved.id)?.mode).toBe('create-destroy');
const off = { ...settings, mode: 'off' };
const paused = scaling.saveScaling(off, saved.id, saved.revision, scaling.previewScaling(off, saved.id, saved.revision).token);
scaling.deleteScaling(saved.id, paused.revision);
tests/unit/autoscaling-dry-run.test.ts
-62
View File
@@ -1,62 +0,0 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { routePermission, roleAllows } from '../../src/lib/server/auth/access.js';
let fixture: Fixture;
beforeEach(async () => {
vi.resetModules(); fixture = await newFixture([]);
const { db } = await import('../../src/lib/server/db/index.js');
for (const n of [3, 8, 9, 10]) db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(`key-${n}`, `secret-${n}`);
});
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); });
const settings = { game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' };
it('persists deduplicated dry-run decisions across restart without provisioning or reservations', async () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const scaling = await import('../../src/lib/server/autoscaling/index.js');
const preview = scaling.previewScaling(settings);
const saved = scaling.saveScaling(settings, undefined, undefined, preview.token);
await scaling.runAutoscaling(); await scaling.runAutoscaling();
closeDb();
await scaling.runAutoscaling();
const view = scaling.scalingReport().policies[0];
expect(view.policy.id).toBe(saved.id);
expect(view.history).toHaveLength(1);
expect(view.decision).toMatchObject({ action: 'create', server: 't6_zombie_town_1' });
expect(JSON.stringify(view)).not.toContain('secret');
for (const table of ['server_deployments', 'native_creation_intents', 'native_instances']) expect(db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).toEqual({ n: 0 });
});
it('rejects duplicate policies, stale edits and changed previews', async () => {
const scaling = await import('../../src/lib/server/autoscaling/index.js');
const preview = scaling.previewScaling(settings);
const saved = scaling.saveScaling(settings, undefined, undefined, preview.token);
expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('already exists');
expect(() => scaling.previewScaling(settings, saved.id, 0)).toThrow('changed');
expect(() => scaling.saveScaling({ ...settings, maximum: 5 }, saved.id, saved.revision, preview.token)).toThrow('preview changed');
const { db } = await import('../../src/lib/server/db/index.js');
const beforeKeyLoss = scaling.previewScaling(settings, saved.id, saved.revision);
db().prepare("DELETE FROM server_keys WHERE id='key-10'").run();
expect(() => scaling.saveScaling(settings, saved.id, saved.revision, beforeKeyLoss.token)).toThrow('preview changed');
expect(scaling.saveScaling(settings, saved.id, saved.revision, scaling.previewScaling(settings, saved.id, saved.revision).token).maximum).toBe(4);
});
it('pauses evaluations and prunes expired observations without deleting current history', async () => {
const scaling = await import('../../src/lib/server/autoscaling/index.js');
scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
await scaling.runAutoscaling();
scaling.setScalingGlobalPause(true); await scaling.runAutoscaling();
expect(scaling.scalingReport().policies[0].decision.reason).toContain('globally paused');
const { db } = await import('../../src/lib/server/db/index.js');
const id = scaling.scalingReport().policies[0].history[0].id;
db().prepare('UPDATE autoscaling_decisions SET last_seen_at=0 WHERE id!=?').run(id);
scaling.pruneAutoscaling();
expect(scaling.scalingReport().policies[0].history).toHaveLength(1);
});
it('uses fleet authorization for page, reads, and mutations', () => {
expect(routePermission('/autoscaling', 'GET')).toBe('viewer');
expect(routePermission('/api/autoscaling', 'GET')).toBe('viewer');
expect(roleAllows('viewer', routePermission('/api/autoscaling', 'POST'))).toBe(false);
expect(roleAllows('operator', routePermission('/api/autoscaling', 'POST'))).toBe(true);
});
tests/unit/autoscaling-game.test.ts
+1 -1
View File
@@ -8,7 +8,7 @@ import { planScaling } from '../../src/lib/server/autoscaling/planner.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { await fixture.cleanup(); });
const settings = { game: 't6', mapSlug: '*', maximum: 2, mode: 'create-only' };
const settings = { game: 't6', mapSlug: '*', maximum: 2, mode: 'create-destroy' };
const maps = zombieMaps('t6');
it('satisfies a T5 game-wide policy with the ten standard Zombies maps', () => {
tests/unit/autoscaling-keys.test.ts
+4 -5
View File
@@ -5,16 +5,16 @@ import { prepareNativeHost } from '../helpers/native-host.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { await fixture.cleanup(); });
const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'dry-run' };
const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-destroy' };
async function setup() {
await prepareNativeHost();
const { db } = await import('../../src/lib/server/db/index.js');
const scaling = await import('../../src/lib/server/autoscaling/index.js');
const { saveServerKey, reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
return { db, scaling, saveServerKey, reserveServerKey };
}
it.each(['off', 'dry-run', 'create-only', 'create-destroy'])('saves a %s policy without preallocating keys for its maximum', async mode => {
if (mode.startsWith('create')) await prepareNativeHost();
it.each(['off', 'create-destroy'])('saves a %s policy without preallocating keys for its maximum', async mode => {
const { db, scaling } = await setup();
const input = { ...settings, mode, maximum: 20 };
const preview = scaling.previewScaling(input);
@@ -65,12 +65,11 @@ it('rechecks game pool deletion and reservations after preview', async () => {
});
it('shares capacity between live map policies and admits the job holding the last key', async () => {
await prepareNativeHost();
const { db, scaling, saveServerKey } = await setup();
saveServerKey('t6', 'game-key');
saveServerKey('t5', 'other-game-key');
for (const mapSlug of ['town', 'farm']) {
const input = { ...settings, mapSlug, mode: 'create-only' };
const input = { ...settings, mapSlug, mode: 'create-destroy' };
scaling.saveScaling(input, undefined, undefined, scaling.previewScaling(input).token);
}
expect(await scaling.runAutoscaling()).toBe(true);
tests/unit/autoscaling-live.test.ts
+5 -2
View File
@@ -5,7 +5,7 @@ import { prepareNativeHost } from '../helpers/native-host.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { await fixture.cleanup(); });
const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-only' };
const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-destroy' };
async function setup() {
await prepareNativeHost();
const { db } = await import('../../src/lib/server/db/index.js');
@@ -77,8 +77,11 @@ it('operator cancellation pauses the policy and does not immediately create a re
it('exact matching cannot remove an owned map absent from the collection', async () => {
const town = 't6_zombie_town_1';
await prepareNativeHost([town]);
const { host } = await prepareNativeHost([town]);
const { db } = await import('../../src/lib/server/db/index.js');
const { autoscalingRemovalCapability } = await import('../../src/lib/server/autoscaling/authorization.js');
db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?")
.run(JSON.stringify([autoscalingRemovalCapability]), host.id);
const at = new Date().toISOString();
db().prepare("INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,0)").run(town, at, at);
for (const n of [1, 2]) db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(`key-${n}`, `key-${n}`);
tests/unit/autoscaling-planner.test.ts
+1 -1
View File
@@ -4,7 +4,7 @@ import { emptyObservation, planScaling } from '../../src/lib/server/autoscaling/
import { mapIdentity } from '../../src/lib/server/autoscaling/snapshot.js';
const now = 2_000_000;
const policy: ScalingPolicy = { ...scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' }), id: 'policy', revision: 1, nextActionAt: 0, pausedReason: null, updatedAt: '' };
const policy: ScalingPolicy = { ...scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4, mode: 'create-destroy' }), id: 'policy', revision: 1, nextActionAt: 0, pausedReason: null, updatedAt: '' };
const name = (n: number) => `t6_zombie_town_${n}`;
const member = (n: number, state: ScalingMember['state'] = 'empty'): ScalingMember => ({ name: name(n), identity: `instance:${n}`, state, players: state === 'occupied' ? 1 : state === 'empty' ? 0 : null, observedAt: now, source: 'rcon', emptySince: 1, blocker: null, operationId: null });
const snapshot = (...members: ScalingMember[]): ScalingSnapshot => ({ members, reservedNames: members.map(m => m.name), keyPool: { total: 4, available: 4 - members.length }, blockers: [] });
tests/unit/autoscaling-policies.test.ts
+116
View File
@@ -0,0 +1,116 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { prepareNativeHost } from '../helpers/native-host.js';
import { routePermission, roleAllows } from '../../src/lib/server/auth/access.js';
let fixture: Fixture;
beforeEach(async () => {
vi.resetModules(); fixture = await newFixture([]);
await prepareNativeHost();
const { db } = await import('../../src/lib/server/db/index.js');
for (const n of [3, 8, 9, 10]) db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(`key-${n}`, `secret-${n}`);
});
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); });
const settings = { game: 't6', mapSlug: 'town', maximum: 4, mode: 'create-destroy' };
it('persists deduplicated paused decisions across restart without provisioning or reservations', async () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const scaling = await import('../../src/lib/server/autoscaling/index.js');
const preview = scaling.previewScaling(settings);
const saved = scaling.saveScaling(settings, undefined, undefined, preview.token);
scaling.setScalingGlobalPause(true);
await scaling.runAutoscaling(); await scaling.runAutoscaling();
closeDb();
await scaling.runAutoscaling();
const view = scaling.scalingReport().policies[0];
expect(view.policy.id).toBe(saved.id);
expect(view.history).toHaveLength(1);
expect(view.decision).toMatchObject({ action: 'wait', server: null });
expect(JSON.stringify(view)).not.toContain('secret');
for (const table of ['server_deployments', 'native_creation_intents', 'native_instances']) expect(db().prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()).toEqual({ n: 0 });
});
it('rejects duplicate policies, stale edits and changed previews', async () => {
const scaling = await import('../../src/lib/server/autoscaling/index.js');
const preview = scaling.previewScaling(settings);
const saved = scaling.saveScaling(settings, undefined, undefined, preview.token);
expect(() => scaling.saveScaling(settings, undefined, undefined, preview.token)).toThrow('overlapping policy');
expect(() => scaling.previewScaling(settings, saved.id, 0)).toThrow('changed');
expect(() => scaling.saveScaling({ ...settings, maximum: 5 }, saved.id, saved.revision, preview.token)).toThrow('preview changed');
const { db } = await import('../../src/lib/server/db/index.js');
const beforeKeyLoss = scaling.previewScaling(settings, saved.id, saved.revision);
db().prepare("DELETE FROM server_keys WHERE id='key-10'").run();
expect(() => scaling.saveScaling(settings, saved.id, saved.revision, beforeKeyLoss.token)).toThrow('preview changed');
expect(scaling.saveScaling(settings, saved.id, saved.revision, scaling.previewScaling(settings, saved.id, saved.revision).token).maximum).toBe(4);
});
it('pauses evaluations and prunes expired observations without deleting current history', async () => {
const scaling = await import('../../src/lib/server/autoscaling/index.js');
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('DELETE FROM server_keys').run();
scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
await scaling.runAutoscaling();
scaling.setScalingGlobalPause(true); await scaling.runAutoscaling();
expect(scaling.scalingReport().policies[0].decision.reason).toContain('globally paused');
const id = scaling.scalingReport().policies[0].history[0].id;
db().prepare('UPDATE autoscaling_decisions SET last_seen_at=0 WHERE id!=?').run(id);
scaling.pruneAutoscaling();
expect(scaling.scalingReport().policies[0].history).toHaveLength(1);
});
it('uses fleet authorization for page, reads, and mutations', () => {
expect(routePermission('/autoscaling', 'GET')).toBe('viewer');
expect(routePermission('/api/autoscaling', 'GET')).toBe('viewer');
expect(roleAllows('viewer', routePermission('/api/autoscaling', 'POST'))).toBe(false);
expect(roleAllows('operator', routePermission('/api/autoscaling', 'POST'))).toBe(true);
});
it.each(['dry-run', 'create-only'])('rejects removed %s mode through the API', async mode => {
const { POST } = await import('../../src/routes/api/autoscaling/+server.js');
for (const action of ['preview', 'save']) {
const response = await POST({ request: new Request('http://localhost/api/autoscaling', {
method: 'POST', body: JSON.stringify({ action, settings: { ...settings, mode } })
}) });
expect(response.status).toBe(400);
expect(await response.json()).toMatchObject({ error: 'Choose a valid autoscaling mode.' });
}
});
it.each(['dry-run', 'create-only'])('turns saved %s policies off once on upgrade', async mode => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const scaling = await import('../../src/lib/server/autoscaling/index.js');
const saved = scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
const otherSettings = { ...settings, mapSlug: 'farm' };
const other = scaling.saveScaling(otherSettings, undefined, undefined, scaling.previewScaling(otherSettings).token);
db().prepare('UPDATE autoscaling_policies SET mode=?,settings=json_set(settings,\'$.mode\',?) WHERE id=?').run(mode, mode, saved.id);
closeDb();
const migrated = scaling.getScalingPolicy(saved.id)!;
expect(migrated).toMatchObject({ mode: 'off', revision: saved.revision + 1, maximum: saved.maximum,
pausedReason: expect.stringContaining('mode was removed') });
expect(db().prepare('SELECT owns_map,json_extract(settings,\'$.mode\') AS mode FROM autoscaling_policies WHERE id=?').get(saved.id))
.toEqual({ owns_map: 0, mode: 'off' });
expect(scaling.getScalingPolicy(other.id)).toEqual(other);
closeDb();
expect(scaling.getScalingPolicy(saved.id)).toEqual(migrated);
const resumed = scaling.saveScaling(settings, migrated.id, migrated.revision,
scaling.previewScaling(settings, migrated.id, migrated.revision).token);
expect(resumed).toMatchObject({ mode: 'create-destroy', pausedReason: null });
});
it.each([false, true])('reconciles removed-mode creation while preserving submitted work: %s', async submitted => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const scaling = await import('../../src/lib/server/autoscaling/index.js');
const saved = scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
await scaling.runAutoscaling();
const { operation_id: id } = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string };
db().prepare("UPDATE autoscaling_policies SET mode='create-only',settings=json_set(settings,'$.mode','create-only') WHERE id=?").run(saved.id);
if (submitted) db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date().toISOString(), id);
closeDb();
expect(scaling.getScalingPolicy(saved.id)?.mode).toBe('off');
expect(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(saved.id)).toEqual({ owns_map: 1 });
expect(await scaling.runAutoscaling()).toBe(false);
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: submitted ? 'queued' : 'cancelled' });
expect(db().prepare('SELECT owns_map FROM autoscaling_policies WHERE id=?').get(saved.id)).toEqual({ owns_map: submitted ? 1 : 0 });
expect(db().prepare('SELECT COUNT(*) AS n FROM server_key_leases').get()).toEqual({ n: submitted ? 1 : 0 });
expect(db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action='create'").get()).toEqual({ n: 1 });
});
tests/unit/autoscaling-snapshot.test.ts
+1 -1
View File
@@ -7,7 +7,7 @@ let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { await fixture.cleanup(); });
const town = 't6_zombie_town_1', farm = 't6_zombie_farm_1';
const policy: ScalingPolicy = { ...scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4, mode: 'dry-run' }), id: 'policy', revision: 1, nextActionAt: 0, pausedReason: null, updatedAt: '' };
const policy: ScalingPolicy = { ...scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4, mode: 'create-destroy' }), id: 'policy', revision: 1, nextActionAt: 0, pausedReason: null, updatedAt: '' };
it('separates variants, deduplicates pending identities, and initializes additive schema', async () => {
await prepareNativeHost([town, farm]);