import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { prepareNativeHost } from '../helpers/native-host.js'; vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kickProvisioning: vi.fn(async () => {}) })); let fixture: Fixture; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); afterEach(async () => { await fixture.cleanup(); }); const names = ['t6_zombie_town_1', 't6_zombie_town_2']; const settings = { game: 't6', mapSlug: 'town', maximum: 3, mode: 'create-destroy' }; async function setup(scope = 'town') { const { host } = await prepareNativeHost(names); const { db } = await import('../../src/lib/server/db/index.js'); const scaling = await import('../../src/lib/server/autoscaling/index.js'); const ownership = await import('../../src/lib/server/autoscaling/ownership.js'); const { autoscalingRemovalCapability } = await import('../../src/lib/server/autoscaling/authorization.js'); db() .prepare( "UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?" ) .run(JSON.stringify([autoscalingRemovalCapability]), host.id); const at = new Date().toISOString(); for (const name of names) { db() .prepare( `INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,?,0)` ) .run(name, at, at, at); db() .prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)") .run(name, `key-${name}`); db().prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)').run(name, name); } db() .prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('spare','t6','spare-key')") .run(); const policy = scaling.saveScaling({ ...settings, mapSlug: scope }); const { POST } = await import('../../src/routes/api/provisioning/+server.js'); const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js'); const { queueFleetAction } = await import('../../src/lib/ui/fleet-actions.js'); const request: typeof fetch = async (input, init) => { const url = new URL(String(input), 'http://localhost'); if (init?.method === 'DELETE') return DELETE({ params: { name: decodeURIComponent(url.pathname.split('/').at(-1)!) } }); return POST({ request: new Request(url, init) }); }; const fleet = (action: 'start' | 'stop' | 'destroy', servers = names) => queueFleetAction(action, servers, request); return { db, scaling, ownership, policy, fleet }; } it.each(['town', '*'])( 'blocks fleet lineup changes while the %s policy is active', async (scope) => { const { db, fleet } = await setup(scope); for (const action of ['start', 'stop', 'destroy'] as const) { expect(await fleet(action)).toEqual({ queued: [], failed: names.map((server) => ({ server, error: expect.stringContaining('Autoscaling') })) }); } expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]); } ); it.each(['start', 'stop', 'destroy'] as const)( 'allows fleet %s during global pause and restores the guard on resume', async (action) => { const { db, scaling, ownership, policy, fleet } = await setup(); scaling.setScalingGlobalPause(true); expect(ownership.scalingOwner('t6', 'town')).toBe(policy.id); expect(ownership.scalingOwnershipBlocker(names[0])).toBeNull(); expect(scaling.evaluateScaling({ ...settings, mapSlug: '*' }).blockers.join(' ')).toContain( 'overlapping' ); expect(() => scaling.saveScaling({ ...settings, mapSlug: '*' })).toThrow('overlapping'); scaling.setScalingGlobalPause(false); expect((await fleet(action)).failed).toHaveLength(2); expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]); scaling.setScalingGlobalPause(true); const result = await fleet(action); expect(result.failed).toEqual([]); expect(result.queued.map((job) => job.server)).toEqual(names); expect( db().prepare('SELECT server,action,state FROM server_deployments ORDER BY server').all() ).toEqual(names.map((server) => ({ server, action, state: 'waiting' }))); } ); it('keeps internal decision authorization scoped to its owning policy during global pause', async () => { const { db, scaling, ownership } = await setup(); const other = scaling.saveScaling({ ...settings, mapSlug: 'farm' }); scaling.setScalingGlobalPause(true); db() .prepare( `INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence,resolved) VALUES('other-decision',?,?,?,?,'fixture','{}','{}',0)` ) .run(other.id, other.revision, Date.now(), Date.now()); expect(() => ownership.assertScalingOwnership('t6', 'town', 'forged-decision')).toThrow( 'Autoscaling' ); expect(() => ownership.assertScalingOwnership('t6', 'town', 'other-decision')).toThrow( 'Autoscaling' ); const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); expect(() => destroyServer(names[0], 'autoscaling:test', 'other-decision')).toThrow( 'Autoscaling' ); expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]); }); it.each(['off', 'paused reason'])( 'permits manual requests while an individually paused policy retains ownership: %s', async (pause) => { const { db, ownership, policy, fleet } = await setup(); // Ownership may remain while a paused policy still has work to reconcile. if (pause === 'off') db().prepare("UPDATE autoscaling_policies SET mode='off' WHERE id=?").run(policy.id); else db() .prepare( "UPDATE autoscaling_policies SET paused_reason='Inspect the failed operation.' WHERE id=?" ) .run(policy.id); expect(ownership.scalingOwner('t6', 'town')).toBe(policy.id); expect(ownership.scalingOwnershipBlocker(names[0])).toBeNull(); expect(() => ownership.assertScalingOwnership('t6', 'town', 'invalid-decision')).toThrow( 'Autoscaling' ); expect((await fleet('stop', [names[0]])).failed).toEqual([]); expect((await fleet('destroy', [names[1]])).failed).toEqual([]); const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js'); expect( queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town' }) ).toMatchObject({ ok: true, name: 't6_zombie_town_3' }); } ); it('rechecks autoscaling when submitting a manual creation preview made during global pause', async () => { const { db, scaling } = await setup(); const { previewCreation, queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js'); const input = { game: 't6' as const, gameMode: 'zombie' as const, mapSlug: 'town' }; expect(queueCreateServer(input)).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') }); scaling.setScalingGlobalPause(true); const preview = previewCreation(input); expect(preview).toMatchObject({ ready: true, name: 't6_zombie_town_3' }); scaling.setScalingGlobalPause(false); expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') }); expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]); scaling.setScalingGlobalPause(true); expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: true, name: preview.name }); }); it.each(['fill', 'exact'] as const)( 'allows collection %s during global pause and rejects a paused preview after resume', async (mode) => { const { db, scaling } = await setup(); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const blueprint = saveBlueprint({ name: 'Town lineup', entries: [{ game: 't6', mapSlug: 'town', count: mode === 'fill' ? 3 : 1 }] }); expect(previewCollection(blueprint.id, mode).blockers.join(' ')).toContain('Autoscaling'); scaling.setScalingGlobalPause(true); const preview = previewCollection(blueprint.id, mode); expect(preview).toMatchObject({ ready: true, blockers: [] }); if (mode === 'fill') expect(preview.servers.map((server) => server.name)).toEqual(['t6_zombie_town_3']); else expect(preview.removals.map((server) => server.name)).toEqual([names[1]]); scaling.setScalingGlobalPause(false); expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling'); expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]); scaling.setScalingGlobalPause(true); expect(queueCollection({ previewId: preview.id })).toMatchObject({ ok: true, collectionId: expect.any(String) }); expect(db().prepare('SELECT server,action FROM server_deployments').all()).toEqual([ { server: mode === 'fill' ? 't6_zombie_town_3' : names[1], action: mode === 'fill' ? 'create' : 'destroy' } ]); } ); it('preserves provisioning conflicts while letting other paused fleet members proceed', async () => { const { db, scaling, fleet } = await setup(); const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); const pending = requestLifecycle(names[0], 'restart'); expect(pending.ok).toBe(true); scaling.setScalingGlobalPause(true); expect(await fleet('stop', [names[0]])).toEqual({ queued: [], failed: [ { server: names[0], error: expect.stringContaining('Another provisioning operation') } ] }); const result = await fleet('destroy'); expect(result.failed).toEqual([ { server: names[0], error: expect.stringContaining('current provisioning operation') } ]); expect(result.queued).toEqual([{ server: names[1], deploymentId: expect.any(String) }]); expect( db().prepare('SELECT server,action FROM server_deployments ORDER BY server').all() ).toEqual([ { server: names[0], action: 'restart' }, { server: names[1], action: 'destroy' } ]); });