Wait for round two before adding spare lobbies

tests/unit/autoscaling-game.test.ts

@@ -35,6 +35,7 @@ it('satisfies a T5 game-wide policy with the ten standard Zombies maps', () => {
         identity: map.slug,
         state: 'empty',
         players: 0,
+        round: null,
         observedAt: now,
         source: 'rcon',
         emptySince: 1,
@@ -84,6 +85,7 @@ it('satisfies a T6 game-wide policy with nine co-op maps and rejects Grief and T
         identity: map.slug,
         state: 'empty',
         players: 0,
+        round: null,
         observedAt: now,
         source: 'rcon',
         emptySince: 1,
@@ -132,6 +134,7 @@ it('applies targets and spare deficits independently to every map', () => {
             identity: `${map.slug}:${n}`,
             state: map.slug === 'town' ? 'occupied' : 'empty',
             players: map.slug === 'town' ? 1 : 0,
+            round: map.slug === 'town' ? 2 : null,
             observedAt: now,
             source: 'rcon',
             emptySince: 1,
@@ -154,6 +157,14 @@ it('applies targets and spare deficits independently to every map', () => {
     });
     snapshot.members = members.filter((m) => m.name !== 't6_zombie_town_2');
     snapshot.reservedNames = snapshot.members.map((m) => m.name);
+    const town = snapshot.members.find((m) => m.name === 't6_zombie_town_1')!;
+    town.round = 1;
+    expect(planScaling(policy, snapshot, now)).toMatchObject({
+        action: 'destroy',
+        server: 't6_zombie_transit_2',
+        deficit: 1
+    });
+    town.round = 2;
     expect(planScaling(policy, snapshot, now)).toMatchObject({
         action: 'create',
         server: 't6_zombie_town_2'

tests/unit/autoscaling-lifecycle.test.ts

@@ -1,6 +1,7 @@
 import { afterEach, beforeEach, expect, it, vi } from 'vitest';
 import { newFixture, type Fixture } from '../helpers/fixture.js';
 import { prepareNativeHost } from '../helpers/native-host.js';
+import { gameEvent } from '../helpers/telemetry.js';
 import { zombieMaps } from '../../src/lib/map-aliases.js';
 
 let fixture: Fixture;
@@ -137,8 +138,15 @@ it.each([
             pausedReason: null
         });
 
-        // Once recovered, actual occupancy can request a new spare normally.
+        // Recovery and occupancy alone cannot request a spare before round two.
         db().prepare('UPDATE servers SET last_players=1 WHERE name=?').run(server);
+        expect(await scaling.runAutoscaling()).toBe(false);
+        vi.setSystemTime(Date.now() + 1_000);
+        const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
+        saveTelemetry({
+            ...gameEvent('heartbeat', 2, 1, server, 'zm_transit'),
+            receivedAt: new Date().toISOString()
+        });
         expect(await scaling.runAutoscaling()).toBe(true);
         expect(
             db().prepare("SELECT server FROM server_deployments WHERE action='create'").all()

tests/unit/autoscaling-live.test.ts

@@ -1,6 +1,7 @@
 import { afterEach, beforeEach, expect, it, vi } from 'vitest';
 import { newFixture, type Fixture } from '../helpers/fixture.js';
 import { prepareNativeHost } from '../helpers/native-host.js';
+import { gameEvent } from '../helpers/telemetry.js';
 
 let fixture: Fixture;
 beforeEach(async () => {
@@ -8,21 +9,122 @@ beforeEach(async () => {
     fixture = await newFixture([]);
 });
 afterEach(async () => {
+    vi.useRealTimers();
     await fixture.cleanup();
 });
 const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-destroy' };
-async function setup() {
-    await prepareNativeHost();
+const occupiedServer = 't6_zombie_town_1';
+async function setup(servers: string[] = []) {
+    const { host } = await prepareNativeHost(servers);
     const { db } = await import('../../src/lib/server/db/index.js');
     const scaling = await import('../../src/lib/server/autoscaling/index.js');
+    const { autoscalingRemovalCapability } =
+        await import('../../src/lib/server/autoscaling/authorization.js');
+    db()
+        .prepare(
+            "UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?"
+        )
+        .run(JSON.stringify([autoscalingRemovalCapability]), host.id);
     for (const n of [3, 8])
         db()
             .prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)")
             .run(`key-${n}`, `key-${n}`);
+    const at = new Date().toISOString();
+    for (const server of servers) {
+        db()
+            .prepare(
+                "INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,0)"
+            )
+            .run(server, at, at);
+        db()
+            .prepare("INSERT INTO server_key_leases(server_id,key_id) VALUES(?,'key-3')")
+            .run(server);
+    }
     const saved = scaling.saveScaling(settings);
     return { db, scaling, saved };
 }
 
+async function report(event: Parameters<typeof gameEvent>[0], round: number, players = 1) {
+    const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
+    vi.setSystemTime(Date.now() + 1_000);
+    saveTelemetry({
+        ...gameEvent(event, round, players, occupiedServer, 'zm_transit'),
+        receivedAt: new Date().toISOString()
+    });
+}
+
+it('leaves jobs and keys untouched when players join and leave before round two', async () => {
+    vi.useFakeTimers({ toFake: ['Date'] });
+    const { db, scaling } = await setup([occupiedServer]);
+    for (const [event, round, players] of [
+        ['game_start', 0, 1],
+        ['heartbeat', 1, 1],
+        ['heartbeat', 1, 0]
+    ] as const) {
+        await report(event, round, players);
+        expect(await scaling.runAutoscaling()).toBe(false);
+        expect(await scaling.runAutoscaling()).toBe(false);
+        expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
+        expect(db().prepare('SELECT server_id FROM server_key_leases').all()).toEqual([
+            { server_id: occupiedServer }
+        ]);
+    }
+});
+
+it.each([2, 17])(
+    'queues exactly one spare once an occupied lobby reaches round %i',
+    async (round) => {
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const { db, scaling } = await setup([occupiedServer]);
+        await report('heartbeat', 1);
+        expect(await scaling.runAutoscaling()).toBe(false);
+        await report('round_change', round);
+        expect(await scaling.runAutoscaling()).toBe(true);
+        await Promise.all([scaling.runAutoscaling(), scaling.runAutoscaling()]);
+        expect(
+            db().prepare("SELECT server,state FROM server_deployments WHERE action='create'").all()
+        ).toEqual([{ server: 't6_zombie_town_2', state: 'queued' }]);
+        expect(db().prepare('SELECT COUNT(*) AS n FROM server_key_leases').get()).toEqual({ n: 2 });
+        const { operation_id: id } = db()
+            .prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0')
+            .get() as { operation_id: string };
+        const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
+        expect(scalingAdmission(id)).toBeNull();
+    }
+);
+
+it.each([
+    ['a new game starts', 'game_start', 0, 1],
+    ['the round resets', 'heartbeat', 1, 1],
+    ['the game ends', 'game_end', 2, 1],
+    ['players leave', 'heartbeat', 2, 0]
+] as const)(
+    'withdraws a queued spare before dispatch when %s',
+    async (_, event, round, players) => {
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const { db, scaling } = await setup([occupiedServer]);
+        await report('round_change', 2);
+        expect(await scaling.runAutoscaling()).toBe(true);
+        const { operation_id: id } = db()
+            .prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0')
+            .get() as { operation_id: string };
+        await report(event, round, players);
+        const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
+        expect(scalingAdmission(id)).toContain('no longer eligible');
+        expect(await scaling.runAutoscaling()).toBe(false);
+        expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({
+            state: 'cancelled'
+        });
+        expect(db().prepare('SELECT server_id FROM server_key_leases').all()).toEqual([
+            { server_id: occupiedServer }
+        ]);
+        expect(await scaling.runAutoscaling()).toBe(false);
+        expect(
+            db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action='create'").get()
+        ).toEqual({ n: 1 });
+    }
+);
+
 it('admits one durable keyed create and reuses it across ticks and restart', async () => {
     const { db, scaling, saved } = await setup();
     expect(await scaling.runAutoscaling()).toBe(true);

tests/unit/autoscaling-planner.test.ts

@@ -22,6 +22,7 @@ const member = (n: number, state: ScalingMember['state'] = 'empty'): ScalingMemb
     identity: `instance:${n}`,
     state,
     players: state === 'occupied' ? 1 : state === 'empty' ? 0 : null,
+    round: state === 'occupied' ? 2 : null,
     observedAt: now,
     source: 'rcon',
     emptySince: 1,
@@ -49,7 +50,7 @@ describe('autoscaling policy planning', () => {
             expect(() => scalingSettings(input)).toThrow();
         expect(scalingSettings({ game: 't6', mapSlug: 'town', maximum: 4 }).mode).toBe('off');
     });
-    it('creates a spare even with free player slots and bootstraps an empty map', () => {
+    it('creates a spare at round two even with free player slots and bootstraps an empty map', () => {
         expect(planScaling(policy, snapshot(member(1, 'occupied')), now)).toMatchObject({
             action: 'create',
             server: name(2),
@@ -63,6 +64,48 @@ describe('autoscaling policy planning', () => {
             'wait'
         );
     });
+    it.each([null, 0, 1])('waits for round two when the current round is %s', (round) => {
+        expect(
+            planScaling(policy, snapshot({ ...member(1, 'occupied'), round }), now)
+        ).toMatchObject({
+            action: 'wait',
+            server: null,
+            desired: 2,
+            deficit: 1,
+            occupied: 1,
+            reason: 'Waiting for occupied lobbies to reach round 2 before adding a spare.'
+        });
+        expect(
+            planScaling(policy, snapshot({ ...member(1, 'occupied'), round }, member(2)), now)
+        ).toMatchObject({ action: 'wait', desired: 2, deficit: 0 });
+    });
+    it.each([2, 10])('replenishes a spare when each occupied lobby reaches round %i', (round) => {
+        const established = { ...member(1, 'occupied'), round: 10 };
+        expect(
+            planScaling(policy, snapshot(established, { ...member(2, 'occupied'), round: 1 }), now)
+        ).toMatchObject({ action: 'wait', desired: 3 });
+        expect(
+            planScaling(policy, snapshot(established, { ...member(2, 'occupied'), round }), now)
+        ).toMatchObject({ action: 'create', server: name(3), desired: 3 });
+    });
+    it('uses established occupancy regardless of lobby number', () => {
+        expect(planScaling(policy, snapshot(member(4, 'occupied')), now)).toMatchObject({
+            action: 'create',
+            server: name(1)
+        });
+    });
+    it.each([{ minimum: 3 }, { spares: 2 }])(
+        'maintains baseline capacity before round two: %j',
+        (settings) => {
+            expect(
+                planScaling(
+                    { ...policy, ...settings },
+                    snapshot({ ...member(1, 'occupied'), round: 1 }),
+                    now
+                )
+            ).toMatchObject({ action: 'create', server: name(2) });
+        }
+    );
     it('selects the first free identity independently of the game key pool size', () => {
         expect(
             planScaling(

tests/unit/autoscaling-snapshot.test.ts

@@ -1,7 +1,9 @@
 import { afterEach, beforeEach, expect, it, vi } from 'vitest';
 import { newFixture, type Fixture } from '../helpers/fixture.js';
 import { prepareNativeHost } from '../helpers/native-host.js';
+import { gameEvent } from '../helpers/telemetry.js';
 import { scalingSettings, type ScalingPolicy } from '../../src/lib/autoscaling.js';
+import type { GameEvent } from '../../src/lib/telemetry.js';
 
 let fixture: Fixture;
 beforeEach(async () => {
@@ -9,6 +11,7 @@ beforeEach(async () => {
     fixture = await newFixture([]);
 });
 afterEach(async () => {
+    vi.useRealTimers();
     await fixture.cleanup();
 });
 const town = 't6_zombie_town_1',
@@ -56,9 +59,90 @@ it('separates variants, deduplicates pending identities, and initializes additiv
     const snapshot = scalingSnapshot(policy);
     expect(snapshot.members).toHaveLength(1);
     expect(snapshot.members[0].state).toBe('creating');
+    expect(snapshot.members[0].round).toBeNull();
     expect(snapshot.reservedNames).toEqual([town]);
 });
 
+async function occupiedLobby() {
+    vi.useFakeTimers({ toFake: ['Date'] });
+    await prepareNativeHost([town]);
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const { scalingSnapshot } = await import('../../src/lib/server/autoscaling/snapshot.js');
+    const now = Date.now();
+    const iso = (offset = 0) => new Date(now + offset).toISOString();
+    db()
+        .prepare(
+            `INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players,active_since,game_started_at)
+            VALUES(?,'t6','zm_transit','town','running',?,?,?,1,?,?)`
+        )
+        .run(town, iso(), iso(), iso(), iso(-120_000), iso(-60_000));
+    const report = (event: GameEvent['event'], round = 2, offset = -1000, players = 1) => {
+        const payload = {
+            ...gameEvent(event, round, players, town, 'zm_transit'),
+            receivedAt: iso(offset)
+        };
+        db()
+            .prepare(
+                `INSERT OR REPLACE INTO server_telemetry(server,round,players,received_at,payload) VALUES(?,?,?,?,?)`
+            )
+            .run(town, round, players, payload.receivedAt, JSON.stringify(payload));
+    };
+    return { db, now, iso, report, member: () => scalingSnapshot(policy, now).members[0] };
+}
+
+it.each(['round_change', 'heartbeat'] as const)(
+    'exposes the current occupied round from fresh %s reports',
+    async (event) => {
+        const { member, report } = await occupiedLobby();
+        expect(member()).toMatchObject({ state: 'occupied', source: 'rcon', round: null });
+        report(event, 1);
+        expect(member()).toMatchObject({ state: 'occupied', round: 1 });
+        report(event, 2);
+        expect(member()).toMatchObject({ state: 'occupied', round: 2 });
+    }
+);
+
+it('does not borrow stale or future rounds while fresh RCON still proves occupancy', async () => {
+    const { member, report } = await occupiedLobby();
+    for (const offset of [-91_000, 500]) {
+        report('heartbeat', 12, offset);
+        expect(member()).toMatchObject({ state: 'occupied', source: 'rcon', round: null });
+    }
+});
+
+it('clears round evidence at match boundaries and when the lobby empties', async () => {
+    const { db, member, report } = await occupiedLobby();
+    report('heartbeat', 12);
+    expect(member().round).toBe(12);
+    for (const event of ['game_start', 'game_end'] as const) {
+        report(event, 12);
+        expect(member()).toMatchObject({ state: 'occupied', round: null });
+    }
+    report('heartbeat', 12, -1000, 0);
+    // A positive RCON safety veto must not turn an empty report's round into demand.
+    expect(member()).toMatchObject({ state: 'occupied', round: null });
+    db().prepare('UPDATE servers SET last_players=0 WHERE name=?').run(town);
+    expect(member()).toMatchObject({ state: 'empty', round: null });
+});
+
+it('rejects round evidence from before the current occupancy, match, or instance lifecycle', async () => {
+    const { db, iso, member, report } = await occupiedLobby();
+    report('heartbeat', 12);
+    for (const field of ['active_since', 'game_started_at', 'game_ended_at']) {
+        db().prepare(`UPDATE servers SET ${field}=? WHERE name=?`).run(iso(), town);
+        expect(member()).toMatchObject({ state: 'occupied', round: null });
+        db()
+            .prepare(`UPDATE servers SET ${field}=? WHERE name=?`)
+            .run(field === 'game_ended_at' ? null : iso(-120_000), town);
+    }
+    db()
+        .prepare('UPDATE native_instances SET generation_reconciled_at=? WHERE server=?')
+        .run(iso(-500), town);
+    expect(member()).toMatchObject({ state: 'occupied', source: 'rcon', round: null });
+    report('heartbeat', 2, 0);
+    expect(member().round).toBe(2);
+});
+
 it('does not create schema mutations or host work when planning an empty map', async () => {
     const { db } = await import('../../src/lib/server/db/index.js');
     const { scalingSnapshot } = await import('../../src/lib/server/autoscaling/snapshot.js');

tests/unit/ui-autoscaling.test.ts

@@ -35,6 +35,7 @@ function view(): ScalingView {
                     identity: 'one',
                     state: 'empty',
                     players: 0,
+                    round: null,
                     observedAt: now,
                     source: 'rcon',
                     emptySince: now,
@@ -47,6 +48,7 @@ function view(): ScalingView {
                     identity: 'two',
                     state: 'creating',
                     players: null,
+                    round: null,
                     observedAt: null,
                     source: null,
                     emptySince: null,