Require revocable authorization for automatic lobby removal

2026-10-06 23:06:04 -04:00
parent e012278bd9
commit e271f31325
18 changed files with 357 additions and 27 deletions
@@ -0,0 +1,103 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { prepareNativeHost } from '../helpers/native-host.js';
+
+vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(async () => 0) }));
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.restoreAllMocks(); });
+const names = ['t6_zombie_town_1', 't6_zombie_town_2'];
+const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-destroy', emptyMinutes: 1, cooldownMinutes: 1 };
+
+async function setup() {
+    const host = await prepareNativeHost(names);
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const scaling = await import('../../src/lib/server/autoscaling/index.js');
+    const authorization = await import('../../src/lib/server/autoscaling/authorization.js');
+    db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?").run(JSON.stringify([authorization.autoscalingRemovalCapability]), host.host.id);
+    const base = Date.now(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(base);
+    for (const name of names) {
+        db().prepare("INSERT INTO servers(name,game,map,map_slug,container_state,last_players) VALUES(?,'t6','zm_transit','town','running',0)").run(name);
+        db().prepare('INSERT INTO server_keys(server_id,pluto_key) VALUES(?,?)').run(name, `key-${name}`);
+    }
+    const fresh = (offset: number) => {
+        vi.setSystemTime(base + offset); const at = new Date().toISOString();
+        db().prepare('UPDATE servers SET container_checked_at=?,last_seen_at=?,last_checked_at=?').run(at, at, at);
+        db().prepare('UPDATE ec_hosts SET last_heartbeat=?').run(at);
+    };
+    fresh(0);
+    const saved = scaling.saveScaling(settings, undefined, undefined, scaling.previewScaling(settings).token);
+    for (const offset of [0, 30_000, 60_000]) { fresh(offset); await scaling.runAutoscaling(); }
+    const intent = db().prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0').get() as { operation_id: string };
+    expect(intent?.operation_id).toBeTruthy();
+    const backend = await import('../../src/lib/server/backends/native.js');
+    vi.spyOn(backend, 'nativeRequest').mockResolvedValue({ healthy: true, state: 'running', exists: true } as never);
+    const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+    const dispatch = async () => {
+        await processProvisioning();
+        const row = db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(intent.operation_id) as { operation_id: string };
+        expect(row.operation_id).toBeTruthy();
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        const command = getHostOperation(row.operation_id)!;
+        return { id: row.operation_id, payload: command.payload.autoscaling as Record<string, unknown> };
+    };
+    return { db, scaling, authorization, policy: saved, operationId: intent.operation_id, hostId: host.host.id, fresh, dispatch, processProvisioning };
+}
+
+it('removes only after a full observed empty interval and claims authorization once', async () => {
+    const s = await setup();
+    const { enableSafetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js');
+    expect(enableSafetyOverride(s.operationId, { id: 'admin', username: 'admin' }).ok).toBe(false);
+    const command = await s.dispatch();
+    expect(s.authorization.claimRemovalAuthorization('wrong-host', command.id, command.payload).authorized).toBe(false);
+    expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, command.payload)).toMatchObject({ authorized: true, validForMs: 5000 });
+    expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, command.payload).authorized).toBe(false);
+});
+
+it('withdraws a queued removal when a player arrives and restarts the empty interval', async () => {
+    const s = await setup();
+    s.db().prepare('UPDATE servers SET last_players=1 WHERE name=?').run(names[0]);
+    await s.scaling.runAutoscaling();
+    expect(s.db().prepare('SELECT state FROM server_deployments WHERE id=?').get(s.operationId)).toEqual({ state: 'cancelled' });
+    s.db().prepare('UPDATE servers SET last_players=0').run();
+    s.fresh(90_000); await s.scaling.runAutoscaling();
+    expect(s.db().prepare('SELECT COUNT(*) AS n FROM autoscaling_decisions WHERE resolved=0').get()).toEqual({ n: 0 });
+});
+
+it('rechecks controller occupancy after the async probe even before monitoring refreshes', async () => {
+    const s = await setup();
+    const rcon = await import('../../src/lib/server/integrations/rcon.js');
+    vi.mocked(rcon.rconGetPlayerCount).mockResolvedValueOnce(1);
+    await s.processProvisioning();
+    expect(s.db().prepare('SELECT state FROM server_deployments WHERE id=?').get(s.operationId)).toEqual({ state: 'cancelled' });
+    expect(s.db().prepare('SELECT COUNT(*) AS n FROM autoscaling_authorizations').get()).toEqual({ n: 0 });
+});
+
+it('revokes a delivered removal on policy pause without claiming it was undone', async () => {
+    const s = await setup(); const command = await s.dispatch();
+    s.db().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(new Date().toISOString(), command.id);
+    const off = { ...settings, mode: 'off' };
+    s.scaling.saveScaling(off, s.policy.id, s.policy.revision, s.scaling.previewScaling(off, s.policy.id, s.policy.revision).token);
+    expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, command.payload).authorized).toBe(false);
+    expect(s.db().prepare('SELECT owns_map FROM autoscaling_policies').get()).toEqual({ owns_map: 1 });
+    expect(s.db().prepare('SELECT state FROM server_deployments WHERE id=?').get(s.operationId)).toEqual({ state: 'running' });
+});
+
+it('withdraws undelivered commands and reconciles invalidation without a retry or failure pause', async () => {
+    const s = await setup(); const command = await s.dispatch();
+    s.scaling.setScalingGlobalPause(true);
+    expect(s.db().prepare('SELECT state FROM ec_host_operations WHERE id=?').get(command.id)).toEqual({ state: 'failed' });
+    await s.processProvisioning(); await s.scaling.runAutoscaling();
+    expect(s.scaling.getScalingPolicy(s.policy.id)?.pausedReason).toBeNull();
+    expect(s.db().prepare('SELECT COUNT(*) AS n FROM autoscaling_decisions WHERE resolved=0').get()).toEqual({ n: 0 });
+});
+
+it('rejects expired, changed revision, and newly protected removal authorizations', async () => {
+    const s = await setup(); const command = await s.dispatch();
+    expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, { ...command.payload, revision: 999 }).authorized).toBe(false);
+    s.db().prepare("INSERT INTO maintenance_jobs(server,requested_at) VALUES(?,?)").run(names[1], new Date().toISOString());
+    expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, command.payload).authorized).toBe(false);
+    s.db().prepare('DELETE FROM maintenance_jobs').run();
+    s.fresh(130_000);
+    expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, command.payload).authorized).toBe(false);
+});