268 lines
12 KiB
TypeScript
268 lines
12 KiB
TypeScript
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { prepareNativeHost } from '../helpers/native-host.js';
|
|
import { gameEvent } from '../helpers/telemetry.js';
|
|
|
|
let fixture: Fixture;
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture([]);
|
|
});
|
|
afterEach(async () => {
|
|
vi.useRealTimers();
|
|
await fixture.cleanup();
|
|
});
|
|
const settings = { game: 't6', mapSlug: 'town', maximum: 2, mode: 'create-destroy' };
|
|
const occupiedServer = 't6_zombie_town_1';
|
|
async function setup(servers: string[] = []) {
|
|
const { host } = await prepareNativeHost(servers);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const scaling = await import('../../src/lib/server/autoscaling/index.js');
|
|
const { autoscalingRemovalCapability } =
|
|
await import('../../src/lib/server/autoscaling/authorization.js');
|
|
db()
|
|
.prepare(
|
|
"UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?"
|
|
)
|
|
.run(JSON.stringify([autoscalingRemovalCapability]), host.id);
|
|
for (const n of [3, 8])
|
|
db()
|
|
.prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)")
|
|
.run(`key-${n}`, `key-${n}`);
|
|
const at = new Date().toISOString();
|
|
for (const server of servers) {
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,0)"
|
|
)
|
|
.run(server, at, at);
|
|
db()
|
|
.prepare("INSERT INTO server_key_leases(server_id,key_id) VALUES(?,'key-3')")
|
|
.run(server);
|
|
}
|
|
const saved = scaling.saveScaling(settings);
|
|
return { db, scaling, saved };
|
|
}
|
|
|
|
async function report(event: Parameters<typeof gameEvent>[0], round: number, players = 1) {
|
|
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
|
vi.setSystemTime(Date.now() + 1_000);
|
|
saveTelemetry({
|
|
...gameEvent(event, round, players, occupiedServer, 'zm_transit'),
|
|
receivedAt: new Date().toISOString()
|
|
});
|
|
}
|
|
|
|
it('leaves jobs and keys untouched when players join and leave before round two', async () => {
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
const { db, scaling } = await setup([occupiedServer]);
|
|
for (const [event, round, players] of [
|
|
['game_start', 0, 1],
|
|
['heartbeat', 1, 1],
|
|
['heartbeat', 1, 0]
|
|
] as const) {
|
|
await report(event, round, players);
|
|
expect(await scaling.runAutoscaling()).toBe(false);
|
|
expect(await scaling.runAutoscaling()).toBe(false);
|
|
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
|
expect(db().prepare('SELECT server_id FROM server_key_leases').all()).toEqual([
|
|
{ server_id: occupiedServer }
|
|
]);
|
|
}
|
|
});
|
|
|
|
it.each([2, 17])(
|
|
'queues exactly one spare once an occupied lobby reaches round %i',
|
|
async (round) => {
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
const { db, scaling } = await setup([occupiedServer]);
|
|
await report('heartbeat', 1);
|
|
expect(await scaling.runAutoscaling()).toBe(false);
|
|
await report('round_change', round);
|
|
expect(await scaling.runAutoscaling()).toBe(true);
|
|
await Promise.all([scaling.runAutoscaling(), scaling.runAutoscaling()]);
|
|
expect(
|
|
db().prepare("SELECT server,state FROM server_deployments WHERE action='create'").all()
|
|
).toEqual([{ server: 't6_zombie_town_2', state: 'queued' }]);
|
|
expect(db().prepare('SELECT COUNT(*) AS n FROM server_key_leases').get()).toEqual({ n: 2 });
|
|
const { operation_id: id } = db()
|
|
.prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0')
|
|
.get() as { operation_id: string };
|
|
const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
|
|
expect(scalingAdmission(id)).toBeNull();
|
|
}
|
|
);
|
|
|
|
it.each([
|
|
['a new game starts', 'game_start', 0, 1],
|
|
['the round resets', 'heartbeat', 1, 1],
|
|
['the game ends', 'game_end', 2, 1],
|
|
['players leave', 'heartbeat', 2, 0]
|
|
] as const)(
|
|
'withdraws a queued spare before dispatch when %s',
|
|
async (_, event, round, players) => {
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
const { db, scaling } = await setup([occupiedServer]);
|
|
await report('round_change', 2);
|
|
expect(await scaling.runAutoscaling()).toBe(true);
|
|
const { operation_id: id } = db()
|
|
.prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0')
|
|
.get() as { operation_id: string };
|
|
await report(event, round, players);
|
|
const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
|
|
expect(scalingAdmission(id)).toContain('no longer eligible');
|
|
expect(await scaling.runAutoscaling()).toBe(false);
|
|
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({
|
|
state: 'cancelled'
|
|
});
|
|
expect(db().prepare('SELECT server_id FROM server_key_leases').all()).toEqual([
|
|
{ server_id: occupiedServer }
|
|
]);
|
|
expect(await scaling.runAutoscaling()).toBe(false);
|
|
expect(
|
|
db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action='create'").get()
|
|
).toEqual({ n: 1 });
|
|
}
|
|
);
|
|
|
|
it('admits one durable keyed create and reuses it across ticks and restart', async () => {
|
|
const { db, scaling, saved } = await setup();
|
|
expect(await scaling.runAutoscaling()).toBe(true);
|
|
await Promise.all([scaling.runAutoscaling(), scaling.runAutoscaling()]);
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
await scaling.runAutoscaling();
|
|
expect(
|
|
db().prepare("SELECT server,state FROM server_deployments WHERE action='create'").all()
|
|
).toEqual([{ server: 't6_zombie_town_1', state: 'queued' }]);
|
|
expect(
|
|
db().prepare('SELECT COUNT(*) AS n FROM autoscaling_decisions WHERE resolved=0').get()
|
|
).toEqual({ n: 1 });
|
|
const { operation_id: id } = db()
|
|
.prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0')
|
|
.get() as { operation_id: string };
|
|
const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
|
|
expect(scalingAdmission(id)).toBeNull();
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
expect(getDeployment(id)?.sources).toContainEqual(
|
|
expect.objectContaining({ kind: 'autoscaling', href: `/autoscaling#${saved.id}` })
|
|
);
|
|
});
|
|
|
|
it('withdraws an unsubmitted create and returns its game key when the policy pauses', async () => {
|
|
const { db, scaling, saved } = await setup();
|
|
await scaling.runAutoscaling();
|
|
const off = { ...settings, mode: 'off' };
|
|
scaling.saveScaling(off, saved.id, saved.revision);
|
|
expect(
|
|
db().prepare("SELECT state FROM server_deployments WHERE action='create'").get()
|
|
).toEqual({ state: 'cancelled' });
|
|
expect(db().prepare('SELECT owns_map FROM autoscaling_policies').get()).toEqual({
|
|
owns_map: 0
|
|
});
|
|
expect(db().prepare('SELECT COUNT(*) AS n FROM server_key_leases').get()).toEqual({ n: 0 });
|
|
db().prepare('DELETE FROM server_keys').run();
|
|
const current = scaling.getScalingPolicy(saved.id)!;
|
|
const evaluation = scaling.evaluateScaling(settings, saved.id, current.revision);
|
|
expect(evaluation.blockers).toEqual([]);
|
|
expect(evaluation.snapshot.keyPool).toEqual({ total: 0, available: 0 });
|
|
});
|
|
|
|
it('blocks manual lifecycle and collection creation, including stale previews', async () => {
|
|
const { db, scaling, saved } = await setup();
|
|
const { queueCreateServer, destroyServer } =
|
|
await import('../../src/lib/server/provisioning/instances.js');
|
|
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
|
|
expect(queueCreateServer({ game: 't6', gameMode: 'zombie', mapSlug: 'town' })).toMatchObject({
|
|
ok: false,
|
|
message: expect.stringContaining('Autoscaling')
|
|
});
|
|
expect(destroyServer('t6_zombie_town_3')).toMatchObject({
|
|
ok: false,
|
|
message: expect.stringContaining('Autoscaling')
|
|
});
|
|
expect(requestLifecycle('t6_zombie_town_3', 'stop')).toMatchObject({ ok: false, status: 409 });
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, queueCollection } =
|
|
await import('../../src/lib/server/fleet/collections.js');
|
|
const blueprint = saveBlueprint({
|
|
name: 'Town',
|
|
entries: [{ game: 't6', mapSlug: 'town', count: 1 }]
|
|
});
|
|
expect(previewCollection(blueprint.id).blockers.join(' ')).toContain('Autoscaling');
|
|
const off = { ...settings, mode: 'off' };
|
|
scaling.saveScaling(off, saved.id, saved.revision);
|
|
const preview = previewCollection(blueprint.id);
|
|
const current = scaling.getScalingPolicy(saved.id)!;
|
|
scaling.saveScaling(settings, saved.id, current.revision);
|
|
expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling');
|
|
expect(db().prepare('SELECT COUNT(*) AS n FROM server_deployments').get()).toEqual({ n: 0 });
|
|
});
|
|
|
|
it('operator cancellation pauses the policy and does not immediately create a replacement', async () => {
|
|
const { db, scaling } = await setup();
|
|
await scaling.runAutoscaling();
|
|
const { operation_id: id } = db()
|
|
.prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0')
|
|
.get() as { operation_id: string };
|
|
const { cancelProvisioningJob } =
|
|
await import('../../src/lib/server/provisioning/cancellation.js');
|
|
expect(cancelProvisioningJob(id)).toBe(true);
|
|
await scaling.runAutoscaling();
|
|
expect(scaling.scalingReport().policies[0].policy.mode).toBe('off');
|
|
expect(
|
|
db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action='create'").get()
|
|
).toEqual({ n: 1 });
|
|
});
|
|
|
|
it('exact matching cannot remove an owned map absent from the collection', async () => {
|
|
const town = 't6_zombie_town_1';
|
|
const { host } = await prepareNativeHost([town]);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { autoscalingRemovalCapability } =
|
|
await import('../../src/lib/server/autoscaling/authorization.js');
|
|
db()
|
|
.prepare(
|
|
"UPDATE ec_hosts SET inventory=json_set(inventory,'$.capabilities',json(?)) WHERE id=?"
|
|
)
|
|
.run(JSON.stringify([autoscalingRemovalCapability]), host.id);
|
|
const at = new Date().toISOString();
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO servers(name,game,map,map_slug,container_state,container_checked_at,last_seen_at,last_players) VALUES(?,'t6','zm_transit','town','running',?,?,0)"
|
|
)
|
|
.run(town, at, at);
|
|
for (const n of [1, 2])
|
|
db()
|
|
.prepare("INSERT INTO server_keys(id,game,pluto_key) VALUES(?,'t6',?)")
|
|
.run(`key-${n}`, `key-${n}`);
|
|
const scaling = await import('../../src/lib/server/autoscaling/index.js');
|
|
scaling.saveScaling(settings);
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, queueCollection } =
|
|
await import('../../src/lib/server/fleet/collections.js');
|
|
const collection = saveBlueprint({
|
|
name: 'Moon only',
|
|
entries: [{ game: 't5', mapSlug: 'moon', count: 1 }]
|
|
});
|
|
const preview = previewCollection(collection.id, 'exact');
|
|
expect(preview.removals.map((r) => r.name)).toContain(town);
|
|
expect(preview.blockers.join(' ')).toContain('Autoscaling');
|
|
expect(() => queueCollection({ previewId: preview.id })).toThrow('Autoscaling');
|
|
});
|
|
|
|
it('honors global pause before dispatch while recognizing its own reserved key', async () => {
|
|
const { db, scaling } = await setup();
|
|
await scaling.runAutoscaling();
|
|
const { operation_id: id } = db()
|
|
.prepare('SELECT operation_id FROM autoscaling_decisions WHERE resolved=0')
|
|
.get() as { operation_id: string };
|
|
const { scalingAdmission } = await import('../../src/lib/server/autoscaling/admission.js');
|
|
expect(scalingAdmission(id)).toBeNull();
|
|
scaling.setScalingGlobalPause(true);
|
|
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(id)).toEqual({
|
|
state: 'cancelled'
|
|
});
|
|
expect(await scaling.runAutoscaling()).toBe(false);
|
|
});
|