Allow manual fleet actions while autoscaling is paused

2026-10-08 14:47:14 -04:00
parent 62e3397d5c
commit dd29b41f95
3 changed files with 172 additions and 8 deletions
@@ -0,0 +1,147 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import { prepareNativeHost } from '../helpers/native-host.js';
+
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kickProvisioning: vi.fn(async () => {}) }));
+
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { await fixture.cleanup(); });
+const names = ['t6_zombie_town_1', 't6_zombie_town_2'];
+const settings = { game: 't6', mapSlug: 'town', maximum: 3, mode: 'create-destroy' };
+
+async function setup(scope = 'town') {
+    const { host } = await prepareNativeHost(names);
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const scaling = await import('../../src/lib/server/autoscaling/index.js');
+    const ownership = await import('../../src/lib/server/autoscaling/ownership.js');
+    const { autoscalingRemovalCapability } = await import('../../src/lib/server/autoscaling/authorization.js');
+    db().prepare("UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?")
+        .run(JSON.stringify([autoscalingRemovalCapability]), host.id);
+    const at = new Date().toISOString();
+    for (const name of names) {
+        db().prepare(`INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players)
+            VALUES(?,'t6','zm_transit','town','running',?,?,?,0)`).run(name, at, at, at);
+        db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)").run(name, `key-${name}`);
+        db().prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)').run(name, name);
+    }
+    db().prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('spare','t6','spare-key')").run();
+    const policy = scaling.saveScaling({ ...settings, mapSlug: scope });
+    const { POST } = await import('../../src/routes/api/provisioning/+server.js');
+    const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js');
+    const { queueFleetAction } = await import('../../src/lib/ui/fleet-actions.js');
+    const request: typeof fetch = async (input, init) => {
+        const url = new URL(String(input), 'http://localhost');
+        if (init?.method === 'DELETE') return DELETE({ params: { name: decodeURIComponent(url.pathname.split('/').at(-1)!) } });
+        return POST({ request: new Request(url, init) });
+    };
+    const fleet = (action: 'start' | 'stop' | 'destroy', servers = names) => queueFleetAction(action, servers, request);
+    return { db, scaling, ownership, policy, fleet };
+}
+
+it.each(['town', '*'])('blocks fleet lineup changes while the %s policy is active', async scope => {
+    const { db, fleet } = await setup(scope);
+    for (const action of ['start', 'stop', 'destroy'] as const) {
+        expect(await fleet(action)).toEqual({ queued: [], failed: names.map(server => ({ server, error: expect.stringContaining('Autoscaling') })) });
+    }
+    expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
+});
+
+it.each(['start', 'stop', 'destroy'] as const)('allows fleet %s during global pause and restores the guard on resume', async action => {
+    const { db, scaling, ownership, policy, fleet } = await setup();
+    scaling.setScalingGlobalPause(true);
+    expect(ownership.scalingOwner('t6', 'town')).toBe(policy.id);
+    expect(ownership.scalingOwnershipBlocker(names[0])).toBeNull();
+    expect(scaling.evaluateScaling({ ...settings, mapSlug: '*' }).blockers.join(' ')).toContain('overlapping');
+    expect(() => scaling.saveScaling({ ...settings, mapSlug: '*' })).toThrow('overlapping');
+    scaling.setScalingGlobalPause(false);
+    expect((await fleet(action)).failed).toHaveLength(2);
+    expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
+    scaling.setScalingGlobalPause(true);
+    const result = await fleet(action);
+    expect(result.failed).toEqual([]);
+    expect(result.queued.map(job => job.server)).toEqual(names);
+    expect(db().prepare('SELECT server,action,state FROM server_deployments ORDER BY server').all())
+        .toEqual(names.map(server => ({ server, action, state: 'waiting' })));
+});
+
+it('keeps internal decision authorization scoped to its owning policy during global pause', async () => {
+    const { db, scaling, ownership } = await setup();
+    const other = scaling.saveScaling({ ...settings, mapSlug: 'farm' });
+    scaling.setScalingGlobalPause(true);
+    db().prepare(`INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence,resolved)
+        VALUES('other-decision',?,?,?,?,'fixture','{}','{}',0)`).run(other.id, other.revision, Date.now(), Date.now());
+    expect(() => ownership.assertScalingOwnership('t6', 'town', 'forged-decision')).toThrow('Autoscaling');
+    expect(() => ownership.assertScalingOwnership('t6', 'town', 'other-decision')).toThrow('Autoscaling');
+    const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
+    expect(() => destroyServer(names[0], 'autoscaling:test', 'other-decision')).toThrow('Autoscaling');
+    expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
+});
+
+it.each(['off', 'paused reason'])('permits manual requests while an individually paused policy retains ownership: %s', async pause => {
+    const { db, ownership, policy, fleet } = await setup();
+    // Ownership may remain while a paused policy still has work to reconcile.
+    if (pause === 'off') db().prepare("UPDATE autoscaling_policies SET mode='off' WHERE id=?").run(policy.id);
+    else db().prepare("UPDATE autoscaling_policies SET paused_reason='Inspect the failed operation.' WHERE id=?").run(policy.id);
+    expect(ownership.scalingOwner('t6', 'town')).toBe(policy.id);
+    expect(ownership.scalingOwnershipBlocker(names[0])).toBeNull();
+    expect(() => ownership.assertScalingOwnership('t6', 'town', 'invalid-decision')).toThrow('Autoscaling');
+    expect((await fleet('stop', [names[0]])).failed).toEqual([]);
+    expect((await fleet('destroy', [names[1]])).failed).toEqual([]);
+    const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
+    expect(queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town' }))
+        .toMatchObject({ ok: true, name: 't6_zombie_town_3' });
+});
+
+it('rechecks autoscaling when submitting a manual creation preview made during global pause', async () => {
+    const { db, scaling } = await setup();
+    const { previewCreation, queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
+    const input = { game: 't6' as const, gameMode: 'zombie' as const, mapSlug: 'town' };
+    expect(queueCreateServer(input)).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') });
+    scaling.setScalingGlobalPause(true);
+    const preview = previewCreation(input);
+    expect(preview).toMatchObject({ ready: true, name: 't6_zombie_town_3' });
+    scaling.setScalingGlobalPause(false);
+    expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: false, message: expect.stringContaining('Autoscaling') });
+    expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
+    scaling.setScalingGlobalPause(true);
+    expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({ ok: true, name: preview.name });
+});
+
+it.each(['fill', 'exact'] as const)('allows collection %s during global pause and rejects a paused preview after resume', async mode => {
+    const { db, scaling } = await setup();
+    const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
+    const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
+    const blueprint = saveBlueprint({ name: 'Town lineup', entries: [{ game: 't6', mapSlug: 'town', count: mode === 'fill' ? 3 : 1 }] });
+    expect(previewCollection(blueprint.id, mode).blockers.join(' ')).toContain('Autoscaling');
+    scaling.setScalingGlobalPause(true);
+    const preview = previewCollection(blueprint.id, mode);
+    expect(preview).toMatchObject({ ready: true, blockers: [] });
+    if (mode === 'fill') expect(preview.servers.map(server => server.name)).toEqual(['t6_zombie_town_3']);
+    else expect(preview.removals.map(server => server.name)).toEqual([names[1]]);
+    scaling.setScalingGlobalPause(false);
+    expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling');
+    expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
+    scaling.setScalingGlobalPause(true);
+    expect(queueCollection({ previewId: preview.id })).toMatchObject({ ok: true, collectionId: expect.any(String) });
+    expect(db().prepare('SELECT server,action FROM server_deployments').all()).toEqual([
+        { server: mode === 'fill' ? 't6_zombie_town_3' : names[1], action: mode === 'fill' ? 'create' : 'destroy' }
+    ]);
+});
+
+it('preserves provisioning conflicts while letting other paused fleet members proceed', async () => {
+    const { db, scaling, fleet } = await setup();
+    const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+    const pending = requestLifecycle(names[0], 'restart');
+    expect(pending.ok).toBe(true);
+    scaling.setScalingGlobalPause(true);
+    expect(await fleet('stop', [names[0]])).toEqual({ queued: [], failed: [
+        { server: names[0], error: expect.stringContaining('Another provisioning operation') }
+    ] });
+    const result = await fleet('destroy');
+    expect(result.failed).toEqual([{ server: names[0], error: expect.stringContaining('current provisioning operation') }]);
+    expect(result.queued).toEqual([{ server: names[1], deploymentId: expect.any(String) }]);
+    expect(db().prepare('SELECT server,action FROM server_deployments ORDER BY server').all()).toEqual([
+        { server: names[0], action: 'restart' }, { server: names[1], action: 'destroy' }
+    ]);
+});