import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { prepareNativeHost } from '../helpers/native-host.js'; import { gameEvent } from '../helpers/telemetry.js'; let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); }); const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-destroy' }; const occupiedServer = 't6_zombie_town_1'; async function setup(servers: string[] = []) { const { host } = await prepareNativeHost(servers); const { db } = await import('../../src/lib/server/db/index.js'); const scaling = await import('../../src/lib/server/autoscaling/index.js'); const { autoscalingRemovalCapability } = await import('../../src/lib/server/autoscaling/authorization.js'); db() .prepare( "UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?" ) .run(JSON.stringify([autoscalingRemovalCapability]), host.id); for (const n of [3, 8]) db() .prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)") .run(`key-${n}`, `key-${n}`); const at = new Date().toISOString(); for (const server of servers) { db() .prepare( "INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,0)" ) .run(server, at, at); db() .prepare("INSERT INTO server_key_leases(server_id,key_id) VALUES(?,'key-3')") .run(server); } const saved = scaling.saveScaling(settings); return { db, scaling, saved }; } async function report(event: Parameters[0], round: number, players = 1) { const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); vi.setSystemTime(Date.now() + 1_000); saveTelemetry({ ...gameEvent(event, round, players, occupiedServer, 'zm_transit'), receivedAt: new Date().toISOString() }); } it('leaves jobs and keys untouched when players join and leave before round two', async () => { vi.useFakeTimers({ toFake: ['Date'] }); const { db, scaling } = await setup([occupiedServer]); for (const [event, round, players] of [ ['game_start', 0, 1], ['heartbeat', 1, 1], ['heartbeat', 1, 0] ] as const) { await report(event, round, players); expect(await scaling.runAutoscaling()).toBe(false); expect(await scaling.runAutoscaling()).toBe(false); expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]); expect(db().prepare('SELECT server_id FROM server_key_leases').all()).toEqual([ { server_id: occupiedServer } ]); } }); it.each([2, 17])( 'queues exactly one spare once an occupied lobby reaches round %i', async (round) => { vi.useFakeTimers({ toFake: ['Date'] }); const { db, scaling } = await setup([occupiedServer]); await report('heartbeat', 1); expect(await scaling.runAutoscaling()).toBe(false); await report('round_change', round); expect(await scaling.runAutoscaling()).toBe(true); await Promise.all([scaling.runAutoscaling(), scaling.runAutoscaling()]); expect( db().prepare("SELECT server,state FROM server_deployments WHERE action='create'").all() ).toEqual([{ server: 't6_zombie_town_2', state: 'queued' }]); expect(db().prepare('SELECT COUNT(*) AS n FROM server_key_leases').get()).toEqual({ n: 2 }); const { operation_id: id } = db() .prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0') .get() as { operation_id: string }; const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js'); expect(scalingAdmission(id)).toBeNull(); } ); it.each([ ['a new game starts', 'game_start', 0, 1], ['the round resets', 'heartbeat', 1, 1], ['the game ends', 'game_end', 2, 1], ['players leave', 'heartbeat', 2, 0] ] as const)( 'withdraws a queued spare before dispatch when %s', async (_, event, round, players) => { vi.useFakeTimers({ toFake: ['Date'] }); const { db, scaling } = await setup([occupiedServer]); await report('round_change', 2); expect(await scaling.runAutoscaling()).toBe(true); const { operation_id: id } = db() .prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0') .get() as { operation_id: string }; await report(event, round, players); const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js'); expect(scalingAdmission(id)).toContain('no longer eligible'); expect(await scaling.runAutoscaling()).toBe(false); expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'cancelled' }); expect(db().prepare('SELECT server_id FROM server_key_leases').all()).toEqual([ { server_id: occupiedServer } ]); expect(await scaling.runAutoscaling()).toBe(false); expect( db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action='create'").get() ).toEqual({ n: 1 }); } ); it('admits one durable keyed create and reuses it across ticks and restart', async () => { const { db, scaling, saved } = await setup(); expect(await scaling.runAutoscaling()).toBe(true); await Promise.all([scaling.runAutoscaling(), scaling.runAutoscaling()]); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); await scaling.runAutoscaling(); expect( db().prepare("SELECT server,state FROM server_deployments WHERE action='create'").all() ).toEqual([{ server: 't6_zombie_town_1', state: 'queued' }]); expect( db().prepare('SELECT COUNT(*) AS n FROM autoscaling_decisions WHERE resolved=0').get() ).toEqual({ n: 1 }); const { operation_id: id } = db() .prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0') .get() as { operation_id: string }; const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js'); expect(scalingAdmission(id)).toBeNull(); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); expect(getDeployment(id)?.sources).toContainEqual( expect.objectContaining({ kind: 'autoscaling', href: `/autoscaling#${saved.id}` }) ); }); it('withdraws an unsubmitted create and returns its game key when the policy pauses', async () => { const { db, scaling, saved } = await setup(); await scaling.runAutoscaling(); const off = { ...settings, mode: 'off' }; scaling.saveScaling(off, saved.id, saved.revision); expect( db().prepare("SELECT state FROM server_deployments WHERE action='create'").get() ).toEqual({ state: 'cancelled' }); expect(db().prepare('SELECT owns_map FROM autoscaling_policies').get()).toEqual({ owns_map: 0 }); expect(db().prepare('SELECT COUNT(*) AS n FROM server_key_leases').get()).toEqual({ n: 0 }); db().prepare('DELETE FROM server_keys').run(); const current = scaling.getScalingPolicy(saved.id)!; const evaluation = scaling.evaluateScaling(settings, saved.id, current.revision); expect(evaluation.blockers).toEqual([]); expect(evaluation.snapshot.keyPool).toEqual({ total: 0, available: 0 }); }); it('blocks manual lifecycle and collection creation, including stale previews', async () => { const { db, scaling, saved } = await setup(); const { queueCreateServer, destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); expect(queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town' })).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') }); expect(destroyServer('t6_zombie_town_3')).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') }); expect(requestLifecycle('t6_zombie_town_3', 'stop')).toMatchObject({ ok: false, status: 409 }); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const blueprint = saveBlueprint({ name: 'Town', entries: [{ game: 't6', mapSlug: 'town', count: 1 }] }); expect(previewCollection(blueprint.id).blockers.join(' ')).toContain('Autoscaling'); const off = { ...settings, mode: 'off' }; scaling.saveScaling(off, saved.id, saved.revision); const preview = previewCollection(blueprint.id); const current = scaling.getScalingPolicy(saved.id)!; scaling.saveScaling(settings, saved.id, current.revision); expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling'); expect(db().prepare('SELECT COUNT(*) AS n FROM server_deployments').get()).toEqual({ n: 0 }); }); it('operator cancellation pauses the policy and does not immediately create a replacement', async () => { const { db, scaling } = await setup(); await scaling.runAutoscaling(); const { operation_id: id } = db() .prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0') .get() as { operation_id: string }; const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js'); expect(cancelProvisioningJob(id)).toBe(true); await scaling.runAutoscaling(); expect(scaling.scalingReport().policies[0].policy.mode).toBe('off'); expect( db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action='create'").get() ).toEqual({ n: 1 }); }); it('exact matching cannot remove an owned map absent from the collection', async () => { const town = 't6_zombie_town_1'; const { host } = await prepareNativeHost([town]); const { db } = await import('../../src/lib/server/db/index.js'); const { autoscalingRemovalCapability } = await import('../../src/lib/server/autoscaling/authorization.js'); db() .prepare( "UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?" ) .run(JSON.stringify([autoscalingRemovalCapability]), host.id); const at = new Date().toISOString(); db() .prepare( "INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,0)" ) .run(town, at, at); for (const n of [1, 2]) db() .prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)") .run(`key-${n}`, `key-${n}`); const scaling = await import('../../src/lib/server/autoscaling/index.js'); scaling.saveScaling(settings); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const collection = saveBlueprint({ name: 'Moon only', entries: [{ game: 't5', mapSlug: 'moon', count: 1 }] }); const preview = previewCollection(collection.id, 'exact'); expect(preview.removals.map((r) => r.name)).toContain(town); expect(preview.blockers.join(' ')).toContain('Autoscaling'); expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling'); }); it('honors global pause before dispatch while recognizing its own reserved key', async () => { const { db, scaling } = await setup(); await scaling.runAutoscaling(); const { operation_id: id } = db() .prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0') .get() as { operation_id: string }; const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js'); expect(scalingAdmission(id)).toBeNull(); scaling.setScalingGlobalPause(true); expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({ state: 'cancelled' }); expect(await scaling.runAutoscaling()).toBe(false); });