241 lines
11 KiB
TypeScript
241 lines
11 KiB
TypeScript
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { prepareNativeHost } from '../helpers/native-host.js';
|
|
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({
|
|
kickProvisioning: vi.fn(async () => {})
|
|
}));
|
|
|
|
let fixture: Fixture;
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture([]);
|
|
});
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
});
|
|
const names = ['t6_zombie_town_1', 't6_zombie_town_2'];
|
|
const settings = { game: 't6', mapSlug: 'town', maximum: 3, mode: 'create-destroy' };
|
|
|
|
async function setup(scope = 'town') {
|
|
const { host } = await prepareNativeHost(names);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const scaling = await import('../../src/lib/server/autoscaling/index.js');
|
|
const ownership = await import('../../src/lib/server/autoscaling/ownership.js');
|
|
const { autoscalingRemovalCapability } =
|
|
await import('../../src/lib/server/autoscaling/authorization.js');
|
|
db()
|
|
.prepare(
|
|
"UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?"
|
|
)
|
|
.run(JSON.stringify([autoscalingRemovalCapability]), host.id);
|
|
const at = new Date().toISOString();
|
|
for (const name of names) {
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_checked_at,last_players)
|
|
VALUES(?,'t6','zm_transit','town','running',?,?,?,0)`
|
|
)
|
|
.run(name, at, at, at);
|
|
db()
|
|
.prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)")
|
|
.run(name, `key-${name}`);
|
|
db().prepare('INSERT INTO server_key_leases(server_id,key_id) VALUES(?,?)').run(name, name);
|
|
}
|
|
db()
|
|
.prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES('spare','t6','spare-key')")
|
|
.run();
|
|
const policy = scaling.saveScaling({ ...settings, mapSlug: scope });
|
|
const { POST } = await import('../../src/routes/api/provisioning/+server.js');
|
|
const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js');
|
|
const { queueFleetAction } = await import('../../src/lib/ui/fleet-actions.js');
|
|
const request: typeof fetch = async (input, init) => {
|
|
const url = new URL(String(input), 'http://localhost');
|
|
if (init?.method === 'DELETE')
|
|
return DELETE({
|
|
params: { name: decodeURIComponent(url.pathname.split('/').at(-1)!) }
|
|
});
|
|
return POST({ request: new Request(url, init) });
|
|
};
|
|
const fleet = (action: 'start' | 'stop' | 'destroy', servers = names) =>
|
|
queueFleetAction(action, servers, request);
|
|
return { db, scaling, ownership, policy, fleet };
|
|
}
|
|
|
|
it.each(['town', '*'])(
|
|
'blocks fleet lineup changes while the %s policy is active',
|
|
async (scope) => {
|
|
const { db, fleet } = await setup(scope);
|
|
for (const action of ['start', 'stop', 'destroy'] as const) {
|
|
expect(await fleet(action)).toEqual({
|
|
queued: [],
|
|
failed: names.map((server) => ({
|
|
server,
|
|
error: expect.stringContaining('Autoscaling')
|
|
}))
|
|
});
|
|
}
|
|
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
|
}
|
|
);
|
|
|
|
it.each(['start', 'stop', 'destroy'] as const)(
|
|
'allows fleet %s during global pause and restores the guard on resume',
|
|
async (action) => {
|
|
const { db, scaling, ownership, policy, fleet } = await setup();
|
|
scaling.setScalingGlobalPause(true);
|
|
expect(ownership.scalingOwner('t6', 'town')).toBe(policy.id);
|
|
expect(ownership.scalingOwnershipBlocker(names[0])).toBeNull();
|
|
expect(scaling.evaluateScaling({ ...settings, mapSlug: '*' }).blockers.join(' ')).toContain(
|
|
'overlapping'
|
|
);
|
|
expect(() => scaling.saveScaling({ ...settings, mapSlug: '*' })).toThrow('overlapping');
|
|
scaling.setScalingGlobalPause(false);
|
|
expect((await fleet(action)).failed).toHaveLength(2);
|
|
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
|
scaling.setScalingGlobalPause(true);
|
|
const result = await fleet(action);
|
|
expect(result.failed).toEqual([]);
|
|
expect(result.queued.map((job) => job.server)).toEqual(names);
|
|
expect(
|
|
db().prepare('SELECT server,action,state FROM server_deployments ORDER BY server').all()
|
|
).toEqual(names.map((server) => ({ server, action, state: 'waiting' })));
|
|
}
|
|
);
|
|
|
|
it('keeps internal decision authorization scoped to its owning policy during global pause', async () => {
|
|
const { db, scaling, ownership } = await setup();
|
|
const other = scaling.saveScaling({ ...settings, mapSlug: 'farm' });
|
|
scaling.setScalingGlobalPause(true);
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO autoscaling_decisions(id,policy_id,revision,evaluated_at,last_seen_at,fingerprint,decision,evidence,resolved)
|
|
VALUES('other-decision',?,?,?,?,'fixture','{}','{}',0)`
|
|
)
|
|
.run(other.id, other.revision, Date.now(), Date.now());
|
|
expect(() => ownership.assertScalingOwnership('t6', 'town', 'forged-decision')).toThrow(
|
|
'Autoscaling'
|
|
);
|
|
expect(() => ownership.assertScalingOwnership('t6', 'town', 'other-decision')).toThrow(
|
|
'Autoscaling'
|
|
);
|
|
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
expect(() => destroyServer(names[0], 'autoscaling:test', 'other-decision')).toThrow(
|
|
'Autoscaling'
|
|
);
|
|
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
|
});
|
|
|
|
it.each(['off', 'paused reason'])(
|
|
'permits manual requests while an individually paused policy retains ownership: %s',
|
|
async (pause) => {
|
|
const { db, ownership, policy, fleet } = await setup();
|
|
// Ownership may remain while a paused policy still has work to reconcile.
|
|
if (pause === 'off')
|
|
db().prepare("UPDATE autoscaling_policies SET mode='off' WHERE id=?").run(policy.id);
|
|
else
|
|
db()
|
|
.prepare(
|
|
"UPDATE autoscaling_policies SET paused_reason='Inspect the failed operation.' WHERE id=?"
|
|
)
|
|
.run(policy.id);
|
|
expect(ownership.scalingOwner('t6', 'town')).toBe(policy.id);
|
|
expect(ownership.scalingOwnershipBlocker(names[0])).toBeNull();
|
|
expect(() => ownership.assertScalingOwnership('t6', 'town', 'invalid-decision')).toThrow(
|
|
'Autoscaling'
|
|
);
|
|
expect((await fleet('stop', [names[0]])).failed).toEqual([]);
|
|
expect((await fleet('destroy', [names[1]])).failed).toEqual([]);
|
|
const { queueCreateServer } =
|
|
await import('../../src/lib/server/provisioning/instances.js');
|
|
expect(
|
|
queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town' })
|
|
).toMatchObject({ ok: true, name: 't6_zombie_town_3' });
|
|
}
|
|
);
|
|
|
|
it('rechecks autoscaling when submitting a manual creation preview made during global pause', async () => {
|
|
const { db, scaling } = await setup();
|
|
const { previewCreation, queueCreateServer } =
|
|
await import('../../src/lib/server/provisioning/instances.js');
|
|
const input = { game: 't6' as const, gameMode: 'zombie' as const, mapSlug: 'town' };
|
|
expect(queueCreateServer(input)).toMatchObject({
|
|
ok: false,
|
|
message: expect.stringContaining('Autoscaling')
|
|
});
|
|
scaling.setScalingGlobalPause(true);
|
|
const preview = previewCreation(input);
|
|
expect(preview).toMatchObject({ ready: true, name: 't6_zombie_town_3' });
|
|
scaling.setScalingGlobalPause(false);
|
|
expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({
|
|
ok: false,
|
|
message: expect.stringContaining('Autoscaling')
|
|
});
|
|
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
|
scaling.setScalingGlobalPause(true);
|
|
expect(queueCreateServer({ ...input, previewId: preview.id })).toMatchObject({
|
|
ok: true,
|
|
name: preview.name
|
|
});
|
|
});
|
|
|
|
it.each(['fill', 'exact'] as const)(
|
|
'allows collection %s during global pause and rejects a paused preview after resume',
|
|
async (mode) => {
|
|
const { db, scaling } = await setup();
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, queueCollection } =
|
|
await import('../../src/lib/server/fleet/collections.js');
|
|
const blueprint = saveBlueprint({
|
|
name: 'Town lineup',
|
|
entries: [{ game: 't6', mapSlug: 'town', count: mode === 'fill' ? 3 : 1 }]
|
|
});
|
|
expect(previewCollection(blueprint.id, mode).blockers.join(' ')).toContain('Autoscaling');
|
|
scaling.setScalingGlobalPause(true);
|
|
const preview = previewCollection(blueprint.id, mode);
|
|
expect(preview).toMatchObject({ ready: true, blockers: [] });
|
|
if (mode === 'fill')
|
|
expect(preview.servers.map((server) => server.name)).toEqual(['t6_zombie_town_3']);
|
|
else expect(preview.removals.map((server) => server.name)).toEqual([names[1]]);
|
|
scaling.setScalingGlobalPause(false);
|
|
expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling');
|
|
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
|
scaling.setScalingGlobalPause(true);
|
|
expect(queueCollection({ previewId: preview.id })).toMatchObject({
|
|
ok: true,
|
|
collectionId: expect.any(String)
|
|
});
|
|
expect(db().prepare('SELECT server,action FROM server_deployments').all()).toEqual([
|
|
{
|
|
server: mode === 'fill' ? 't6_zombie_town_3' : names[1],
|
|
action: mode === 'fill' ? 'create' : 'destroy'
|
|
}
|
|
]);
|
|
}
|
|
);
|
|
|
|
it('preserves provisioning conflicts while letting other paused fleet members proceed', async () => {
|
|
const { db, scaling, fleet } = await setup();
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const pending = requestLifecycle(names[0], 'restart');
|
|
expect(pending.ok).toBe(true);
|
|
scaling.setScalingGlobalPause(true);
|
|
expect(await fleet('stop', [names[0]])).toEqual({
|
|
queued: [],
|
|
failed: [
|
|
{ server: names[0], error: expect.stringContaining('Another provisioning operation') }
|
|
]
|
|
});
|
|
const result = await fleet('destroy');
|
|
expect(result.failed).toEqual([
|
|
{ server: names[0], error: expect.stringContaining('current provisioning operation') }
|
|
]);
|
|
expect(result.queued).toEqual([{ server: names[1], deploymentId: expect.any(String) }]);
|
|
expect(
|
|
db().prepare('SELECT server,action FROM server_deployments ORDER BY server').all()
|
|
).toEqual([
|
|
{ server: names[0], action: 'restart' },
|
|
{ server: names[1], action: 'destroy' }
|
|
]);
|
|
});
|