Wait for round two before adding spare lobbies

tests/unit/autoscaling-live.test.ts

@@ -1,6 +1,7 @@
 import { afterEach, beforeEach, expect, it, vi } from 'vitest';
 import { newFixture, type Fixture } from '../helpers/fixture.js';
 import { prepareNativeHost } from '../helpers/native-host.js';
+import { gameEvent } from '../helpers/telemetry.js';
 
 let fixture: Fixture;
 beforeEach(async () => {
@@ -8,21 +9,122 @@ beforeEach(async () => {
     fixture = await newFixture([]);
 });
 afterEach(async () => {
+    vi.useRealTimers();
     await fixture.cleanup();
 });
 const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-destroy' };
-async function setup() {
-    await prepareNativeHost();
+const occupiedServer = 't6_zombie_town_1';
+async function setup(servers: string[] = []) {
+    const { host } = await prepareNativeHost(servers);
     const { db } = await import('../../src/lib/server/db/index.js');
     const scaling = await import('../../src/lib/server/autoscaling/index.js');
+    const { autoscalingRemovalCapability } =
+        await import('../../src/lib/server/autoscaling/authorization.js');
+    db()
+        .prepare(
+            "UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?"
+        )
+        .run(JSON.stringify([autoscalingRemovalCapability]), host.id);
     for (const n of [3, 8])
         db()
             .prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)")
             .run(`key-${n}`, `key-${n}`);
+    const at = new Date().toISOString();
+    for (const server of servers) {
+        db()
+            .prepare(
+                "INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,0)"
+            )
+            .run(server, at, at);
+        db()
+            .prepare("INSERT INTO server_key_leases(server_id,key_id) VALUES(?,'key-3')")
+            .run(server);
+    }
     const saved = scaling.saveScaling(settings);
     return { db, scaling, saved };
 }
 
+async function report(event: Parameters<typeof gameEvent>[0], round: number, players = 1) {
+    const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
+    vi.setSystemTime(Date.now() + 1_000);
+    saveTelemetry({
+        ...gameEvent(event, round, players, occupiedServer, 'zm_transit'),
+        receivedAt: new Date().toISOString()
+    });
+}
+
+it('leaves jobs and keys untouched when players join and leave before round two', async () => {
+    vi.useFakeTimers({ toFake: ['Date'] });
+    const { db, scaling } = await setup([occupiedServer]);
+    for (const [event, round, players] of [
+        ['game_start', 0, 1],
+        ['heartbeat', 1, 1],
+        ['heartbeat', 1, 0]
+    ] as const) {
+        await report(event, round, players);
+        expect(await scaling.runAutoscaling()).toBe(false);
+        expect(await scaling.runAutoscaling()).toBe(false);
+        expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
+        expect(db().prepare('SELECT server_id FROM server_key_leases').all()).toEqual([
+            { server_id: occupiedServer }
+        ]);
+    }
+});
+
+it.each([2, 17])(
+    'queues exactly one spare once an occupied lobby reaches round %i',
+    async (round) => {
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const { db, scaling } = await setup([occupiedServer]);
+        await report('heartbeat', 1);
+        expect(await scaling.runAutoscaling()).toBe(false);
+        await report('round_change', round);
+        expect(await scaling.runAutoscaling()).toBe(true);
+        await Promise.all([scaling.runAutoscaling(), scaling.runAutoscaling()]);
+        expect(
+            db().prepare("SELECT server,state FROM server_deployments WHERE action='create'").all()
+        ).toEqual([{ server: 't6_zombie_town_2', state: 'queued' }]);
+        expect(db().prepare('SELECT COUNT(*) AS n FROM server_key_leases').get()).toEqual({ n: 2 });
+        const { operation_id: id } = db()
+            .prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0')
+            .get() as { operation_id: string };
+        const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
+        expect(scalingAdmission(id)).toBeNull();
+    }
+);
+
+it.each([
+    ['a new game starts', 'game_start', 0, 1],
+    ['the round resets', 'heartbeat', 1, 1],
+    ['the game ends', 'game_end', 2, 1],
+    ['players leave', 'heartbeat', 2, 0]
+] as const)(
+    'withdraws a queued spare before dispatch when %s',
+    async (_, event, round, players) => {
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const { db, scaling } = await setup([occupiedServer]);
+        await report('round_change', 2);
+        expect(await scaling.runAutoscaling()).toBe(true);
+        const { operation_id: id } = db()
+            .prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0')
+            .get() as { operation_id: string };
+        await report(event, round, players);
+        const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
+        expect(scalingAdmission(id)).toContain('no longer eligible');
+        expect(await scaling.runAutoscaling()).toBe(false);
+        expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({
+            state: 'cancelled'
+        });
+        expect(db().prepare('SELECT server_id FROM server_key_leases').all()).toEqual([
+            { server_id: occupiedServer }
+        ]);
+        expect(await scaling.runAutoscaling()).toBe(false);
+        expect(
+            db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action='create'").get()
+        ).toEqual({ n: 1 });
+    }
+);
+
 it('admits one durable keyed create and reuses it across ticks and restart', async () => {
     const { db, scaling, saved } = await setup();
     expect(await scaling.runAutoscaling()).toBe(true);