From 62e3397d5c88525f89d8cd694125cf1995c56100 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Thu, 8 Oct 2026 14:41:22 -0400 Subject: [PATCH] Keep lower-numbered lobbies during autoscaling --- docs/guides/autoscaling.md | 8 +++++ src/lib/server/autoscaling/planner.ts | 10 ++++-- tests/unit/autoscaling-game.test.ts | 2 +- tests/unit/autoscaling-planner.test.ts | 45 ++++++++++++++++++++++++-- tests/unit/autoscaling-removal.test.ts | 7 ++-- 5 files changed, 63 insertions(+), 9 deletions(-) diff --git a/docs/guides/autoscaling.md b/docs/guides/autoscaling.md index 93a381e..cec765c 100644 --- a/docs/guides/autoscaling.md +++ b/docs/guides/autoscaling.md @@ -21,6 +21,14 @@ Plutonium keys belong to a game pool shared by every map, manual launch, collect The desired count is `min(maximum, max(minimum, occupied lobbies + empty spare target))`. The controller evaluates policies every 15 seconds and requests at most one unresolved operation per policy. Game-wide policies allow one outstanding operation across their maps, prioritizing additions before removals. Once an operation settles, the next evaluation can act immediately on another eligible lobby. Surplus lobbies must still satisfy the configured continuous empty duration before removal. Create uses the existing startup concurrency, CPU, memory, library, port, and host admission checks. Lack of host capacity leaves one request waiting; it does not create additional requests every tick. Creates use current promoted launch files when provisioning begins. +When trimming a map, autoscaling keeps occupied lobbies and the lowest-numbered +empty lobbies needed for its minimum and spare targets. It selects the +highest-numbered eligible surplus lobby for removal. The required empty lobbies +are protected before checking idle timers, so Lobby 1 stays available while a +higher surplus lobby completes its empty interval. Existing numbering gaps remain +until normal growth needs another lobby; autoscaling does not replace healthy +lobbies just to renumber them. + Existing policies saved in the removed Dry run or Create only modes switch to Off on upgrade. Review and turn autoscaling On to resume; pending work remains tracked until settled. ## Ownership and blockers diff --git a/src/lib/server/autoscaling/planner.ts b/src/lib/server/autoscaling/planner.ts index 49c0fa5..430d6f2 100644 --- a/src/lib/server/autoscaling/planner.ts +++ b/src/lib/server/autoscaling/planner.ts @@ -57,8 +57,14 @@ export function planScaling(policy: ScalingPolicy, snapshot: ScalingSnapshot, no return available ? result('create', 'Replenish empty lobby capacity.', available) : result('wait', 'No unused lobby identity is available.'); } if (occupied + empty > desired && empty > policy.spares && occupied + empty > policy.minimum) { - const candidates = members.filter(m => m.state === 'empty' && m.emptySince !== null && m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000) - .sort((a, b) => a.emptySince! - b.emptySince! || a.name.localeCompare(b.name, 'en', { numeric: true })); + // Protect the lowest-numbered spares before checking removal timers. + // Lobby 1 must survive even when a higher lobby has only just emptied. + const orderedEmpty = members.filter(m => m.state === 'empty').sort((a, b) => + (parseServerName(a.name)?.lobby ?? Number.MAX_SAFE_INTEGER) - (parseServerName(b.name)?.lobby ?? Number.MAX_SAFE_INTEGER) + || a.name.localeCompare(b.name, 'en', { numeric: true })); + const requiredEmpty = Math.max(policy.spares, policy.minimum - occupied); + const candidates = orderedEmpty.slice(requiredEmpty).reverse() + .filter(m => m.emptySince !== null && m.observedAt! - m.emptySince >= policy.emptyMinutes * 60_000); if (candidates.length) return result('destroy', 'Surplus lobby has remained empty.', candidates[0].name); return result('wait', 'Waiting for a continuous empty interval.'); } diff --git a/tests/unit/autoscaling-game.test.ts b/tests/unit/autoscaling-game.test.ts index 96ab15e..d9be50d 100644 --- a/tests/unit/autoscaling-game.test.ts +++ b/tests/unit/autoscaling-game.test.ts @@ -50,7 +50,7 @@ it('applies targets and spare deficits independently to every map', () => { observedAt: now, source: 'rcon', emptySince: 1, blocker: null, operationId: null }))); const snapshot = { members, reservedNames: members.map(m => m.name), keyPool: { total: 15, available: 1 }, blockers: [] }; // Other maps' empty lobbies cannot satisfy Town's spare target or maximum. - expect(planScaling(policy, snapshot, now)).toMatchObject({ action: 'destroy', server: 't6_zombie_transit_1', desired: maps.length + 1, deficit: 1 }); + expect(planScaling(policy, snapshot, now)).toMatchObject({ action: 'destroy', server: 't6_zombie_transit_2', desired: maps.length + 1, deficit: 1 }); snapshot.members = members.filter(m => m.name !== 't6_zombie_town_2'); snapshot.reservedNames = snapshot.members.map(m => m.name); expect(planScaling(policy, snapshot, now)).toMatchObject({ action: 'create', server: 't6_zombie_town_2' }); diff --git a/tests/unit/autoscaling-planner.test.ts b/tests/unit/autoscaling-planner.test.ts index 22562e7..b35234a 100644 --- a/tests/unit/autoscaling-planner.test.ts +++ b/tests/unit/autoscaling-planner.test.ts @@ -25,7 +25,7 @@ describe('autoscaling policy planning', () => { expect(planScaling(policy, { ...snapshot(), keyPool: { total: 2, available: 0 } }, now).reason).toContain('keys are in use'); }); it('still removes eligible surplus lobbies when the game pool is exhausted', () => { - expect(planScaling(policy, { ...snapshot(member(1), member(2)), keyPool: { total: 2, available: 0 } }, now)).toMatchObject({ action: 'destroy', server: name(1) }); + expect(planScaling(policy, { ...snapshot(member(1), member(2)), keyPool: { total: 2, available: 0 } }, now)).toMatchObject({ action: 'destroy', server: name(2) }); }); it('holds for pending creation, unknown, stopped, or protected members', () => { for (const state of ['creating', 'removing', 'unknown', 'stopped', 'protected'] as const) expect(planScaling(policy, snapshot(member(1, state)), now).action).toBe('wait'); @@ -40,10 +40,49 @@ describe('autoscaling policy planning', () => { expect(planScaling({ ...policy, maximum: 1 }, snapshot(member(1, 'occupied'), member(2)), now).action).toBe('wait'); expect(planScaling({ ...policy, minimum: 3 }, snapshot(member(1), member(2), member(3)), now).action).toBe('wait'); }); - it('trims one longest-empty lobby and requires a full observed interval', () => { - expect(planScaling(policy, snapshot(member(1, 'occupied'), member(2), member(3), member(4)), now)).toMatchObject({ action: 'destroy', server: name(2), desired: 2 }); + it('trims one highest-numbered surplus lobby and requires a full observed interval', () => { + expect(planScaling(policy, snapshot(member(1, 'occupied'), member(2), member(3), member(4)), now)).toMatchObject({ action: 'destroy', server: name(4), desired: 2 }); expect(planScaling(policy, snapshot({ ...member(1), emptySince: now }, { ...member(2), emptySince: now }), now).action).toBe('wait'); }); + it('keeps Lobby 1 when it has been empty longer than a newly empty Lobby 2', () => { + const members = [{ ...member(2), emptySince: now }, member(1)]; + expect(planScaling(policy, snapshot(...members), now)).toMatchObject({ action: 'wait', server: null, desired: 1 }); + const later = now + policy.emptyMinutes * 60_000; + expect(planScaling(policy, snapshot(...members.map(m => ({ ...m, observedAt: later }))), later)) + .toMatchObject({ action: 'destroy', server: name(2), desired: 1 }); + }); + it.each([1, 2])('retains the lowest empty spare while Lobby %i is occupied', occupied => { + const members = [member(1, occupied === 1 ? 'occupied' : 'empty'), member(2, occupied === 2 ? 'occupied' : 'empty')]; + expect(planScaling(policy, snapshot(...members), now)).toMatchObject({ action: 'wait', desired: 2 }); + expect(planScaling(policy, snapshot(...members, { ...member(3), emptySince: now }), now)).toMatchObject({ action: 'wait', desired: 2 }); + expect(planScaling(policy, snapshot(...members, member(3)), now)).toMatchObject({ action: 'destroy', server: name(3), desired: 2 }); + }); + it('preserves occupied higher-numbered lobbies while trimming empty surplus', () => { + expect(planScaling(policy, snapshot(member(1), member(2), member(3), member(4, 'occupied')), now)) + .toMatchObject({ action: 'destroy', server: name(3), desired: 2 }); + expect(planScaling(policy, snapshot(member(1), member(4, 'occupied')), now)).toMatchObject({ action: 'wait', desired: 2 }); + }); + it.each([ + { minimum: 3, spares: 1, occupied: null, surplus: 4 }, + { minimum: 3, spares: 1, occupied: 4, surplus: 3 }, + { minimum: 1, spares: 2, occupied: 4, surplus: 3 } + ])('retains low-numbered empties for minimum $minimum and spares $spares with occupied Lobby $occupied', ({ minimum, spares, occupied, surplus }) => { + const settings = { ...policy, minimum, spares }; + const members = [4, 3, 2, 1].map(n => member(n, n === occupied ? 'occupied' : 'empty')); + expect(planScaling(settings, snapshot(...members.map(m => m.name === name(surplus) ? { ...m, emptySince: now } : m)), now)) + .toMatchObject({ action: 'wait', desired: 3 }); + expect(planScaling(settings, snapshot(...members), now)).toMatchObject({ action: 'destroy', server: name(surplus), desired: 3 }); + expect(planScaling(settings, snapshot(...members.filter(m => m.name !== name(surplus))), now)).toMatchObject({ action: 'wait', desired: 3 }); + }); + it('orders lobby numbers numerically when choosing survivors and surplus', () => { + expect(planScaling(policy, snapshot(member(10), member(2)), now)).toMatchObject({ action: 'destroy', server: name(10) }); + expect(planScaling(policy, snapshot(member(2), member(10), member(1)), now)).toMatchObject({ action: 'destroy', server: name(10) }); + }); + it('keeps an existing solo Lobby 2 and fills Lobby 1 when occupancy requires growth', () => { + expect(planScaling(policy, snapshot(member(2)), now)).toMatchObject({ action: 'wait', server: null, desired: 1 }); + expect(planScaling(policy, snapshot(member(2, 'occupied')), now)).toMatchObject({ action: 'create', server: name(1), desired: 2 }); + expect(planScaling(policy, snapshot(member(2), member(1)), now)).toMatchObject({ action: 'destroy', server: name(2), desired: 1 }); + }); it('preserves map variants and refuses ambiguous engine maps', () => { expect(mapIdentity({ name: 'legacy', game: 't6', map: 'zm_transit' })).toBeNull(); expect(mapIdentity({ name: name(1), map: 'zm_transit' })?.mapSlug).toBe('town'); diff --git a/tests/unit/autoscaling-removal.test.ts b/tests/unit/autoscaling-removal.test.ts index 37ac3c7..c03ae22 100644 --- a/tests/unit/autoscaling-removal.test.ts +++ b/tests/unit/autoscaling-removal.test.ts @@ -51,8 +51,9 @@ async function setup(gameWide = false) { return { db, scaling, authorization, policy: saved, operationId: intent.operation_id, hostId: host.host.id, fresh, dispatch, processProvisioning }; } -it('removes only after a full observed empty interval and claims authorization once', async () => { +it('removes the higher surplus lobby after a full empty interval and claims authorization once', async () => { const s = await setup(); + expect(s.db().prepare('SELECT server FROM server_deployments WHERE id=?').get(s.operationId)).toEqual({ server: names[1] }); const { enableSafetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js'); expect(enableSafetyOverride(s.operationId, { id: 'admin', username: 'admin' }).ok).toBe(false); const command = await s.dispatch(); @@ -63,7 +64,7 @@ it('removes only after a full observed empty interval and claims authorization o it('withdraws a queued removal when a player arrives and restarts the empty interval', async () => { const s = await setup(); - s.db().prepare('UPDATE servers SET last_players=1 WHERE name=?').run(names[0]); + s.db().prepare('UPDATE servers SET last_players=1 WHERE name=?').run(names[1]); await s.scaling.runAutoscaling(); expect(s.db().prepare('SELECT state FROM server_deployments WHERE id=?').get(s.operationId)).toEqual({ state: 'cancelled' }); s.db().prepare('UPDATE servers SET last_players=0').run(); @@ -115,7 +116,7 @@ it('rejects expired, changed revision, and newly protected removal authorization it('authorizes a game-wide removal without trimming any map below its targets', async () => { const s = await setup(true); - expect(s.db().prepare("SELECT server FROM server_deployments WHERE id=?").get(s.operationId)).toEqual({ server: names[0] }); + expect(s.db().prepare("SELECT server FROM server_deployments WHERE id=?").get(s.operationId)).toEqual({ server: names[1] }); const command = await s.dispatch(); expect(s.authorization.claimRemovalAuthorization(s.hostId, command.id, command.payload).authorized).toBe(true); });