244 lines
17 KiB
TypeScript
244 lines
17 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
|
rconGetPlayerCount: vi.fn(async () => 0),
|
|
rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })),
|
|
rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: '',
|
|
status: { players: 0, map: 'zm_transit', gametype: null, roster: [] } }))
|
|
}));
|
|
|
|
let fixture: Fixture;
|
|
let hostId: string;
|
|
let agentCredential: string;
|
|
let heartbeatInventory: Record<string, unknown>;
|
|
let releasePlacement: (() => void) | undefined;
|
|
const image = 'example.test/plutainer:test';
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules(); vi.clearAllMocks();
|
|
releasePlacement = undefined;
|
|
fixture = await newFixture([]);
|
|
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
|
|
vi.stubEnv('PLUTAINER_IMAGE', image);
|
|
vi.stubEnv('RCON_PASSWORD', 'test-password');
|
|
vi.stubEnv('KOMODO_ADDRESS', '');
|
|
vi.stubEnv('KOMODO_API_KEY', '');
|
|
vi.stubEnv('KOMODO_API_SECRET', '');
|
|
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
|
const { createHost, createEnrollment, enrollHost, heartbeat, reserveHostPort } = await import('../../src/lib/server/hosts/index.js');
|
|
hostId = createHost({ name: 'Match collection host', portMin: 28960, portMax: 28999 }).id;
|
|
const inventory = {
|
|
cpuCount: 64, cpuUsagePercent: 5, memoryTotal: 128 * 1024 ** 3, memoryAvailable: 120 * 1024 ** 3,
|
|
protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
|
|
storageAvailable: 1_000_000_000_000, runtimeImages: [image]
|
|
};
|
|
agentCredential = enrollHost(createEnrollment(hostId).token, inventory).credential;
|
|
const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const libraries = ['t5', 't6'].map(game => {
|
|
const id = createLibraryImport({ hostId, game }).id;
|
|
db().prepare("UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?")
|
|
.run('a'.repeat(64), new Date().toISOString(), id);
|
|
return { id, state: 'ready' as const };
|
|
});
|
|
updateHostLibraryCaches(hostId, libraries.map(library => ({ libraryVersionId: library.id, status: 'ready', manifestSha256: 'a'.repeat(64) })));
|
|
heartbeatInventory = { ...inventory, libraries };
|
|
heartbeat(hostId, heartbeatInventory);
|
|
const { T6_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
|
|
const now = new Date().toISOString();
|
|
db().transaction(() => {
|
|
for (const [index, map] of T6_ZOMBIE_MAPS.slice(0, 9).entries()) {
|
|
const name = `t6_zombie_${map.slug}_1`;
|
|
const instanceId = randomUUID();
|
|
const deploymentId = randomUUID();
|
|
const port = 28960 + index;
|
|
db().prepare(`INSERT INTO servers(name,komodo_name,game,map,game_mode,map_slug,game_port,container_state,last_players,last_seen_at)
|
|
VALUES(?,'','t6',?,'zombie',?,?,'running',0,?)`).run(name, map.canonical, map.slug, port, now);
|
|
db().prepare('INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,?,?,?,?)')
|
|
.run(instanceId, name, hostId, libraries[1].id, image, port, now);
|
|
db().prepare("INSERT INTO server_deployments(id,server,action,state,stage,triggered_by,started_at,finished_at) VALUES(?,?,'create','done','Verified','fixture',?,?)")
|
|
.run(deploymentId, name, now, now);
|
|
db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(deploymentId, instanceId);
|
|
for (const protocol of ['tcp', 'udp'] as const) reserveHostPort(hostId, instanceId, { port, protocol });
|
|
}
|
|
})();
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => {
|
|
if (kind !== 'instance.inspect') throw new Error(`Unexpected request ${kind}`);
|
|
return { exists: true, healthy: true, state: 'running' } as never;
|
|
});
|
|
});
|
|
|
|
afterEach(async () => {
|
|
releasePlacement?.();
|
|
await new Promise<void>(resolve => setImmediate(resolve));
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
await new Promise<void>(resolve => setImmediate(resolve));
|
|
vi.restoreAllMocks();
|
|
await fixture.cleanup();
|
|
vi.unstubAllEnvs(); vi.unstubAllGlobals();
|
|
});
|
|
|
|
describe('matching a native collection', () => {
|
|
it('places queued creations and completes all exact removals during later worker passes', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 10, destroyConcurrency: 9 });
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
|
const { T5_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
|
|
const blueprint = saveBlueprint({ name: 'Replace nine with ten', entries: T5_ZOMBIE_MAPS.filter(map => map.slug !== 'zombietron')
|
|
.map(map => ({ game: 't5', mapSlug: map.slug, count: 1 })) });
|
|
const preview = previewCollection(blueprint.id, 'exact');
|
|
const queued = queueCollection({ previewId: preview.id,
|
|
keys: Object.fromEntries(preview.servers.map(server => [server.name!, `key-${server.name}`])) });
|
|
expect(queueCollection({ previewId: preview.id })).toEqual(queued);
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toHaveLength(9);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(18);
|
|
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
const operations = db().prepare("SELECT id,kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.remove')")
|
|
.all() as { id: string; kind: string }[];
|
|
expect(operations.filter(operation => operation.kind === 'instance.create')).toHaveLength(10);
|
|
const removals = operations.filter(operation => operation.kind === 'instance.remove');
|
|
expect(removals).toHaveLength(9);
|
|
const { reportHostOperationStatus } = await import('../../src/lib/server/hosts/index.js');
|
|
for (const operation of removals) reportHostOperationStatus(hostId, operation.id, { state: 'completed', result: { exists: false } });
|
|
await processProvisioning();
|
|
const active = db().prepare('SELECT id,port FROM native_instances WHERE retired_at IS NULL').all() as { id: string; port: number }[];
|
|
expect(active).toHaveLength(10);
|
|
expect(new Set(active.map(instance => instance.port)).size).toBe(10);
|
|
for (const instance of active) expect(db().prepare('SELECT protocol,port FROM ec_host_ports WHERE instance_id=? ORDER BY protocol').all(instance.id))
|
|
.toEqual([{ protocol: 'tcp', port: instance.port }, { protocol: 'udp', port: instance.port }]);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(20);
|
|
expect(getCollection(queued.collectionId!)?.servers.filter(server => server.action === 'destroy').map(server => server.state))
|
|
.toEqual(Array(9).fill('done'));
|
|
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.remove')").all()).toHaveLength(19);
|
|
});
|
|
|
|
it('accepts nine removals and ten unplaced creations before capacity checks or file loading', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { createScript, setScope, deleteScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const content = Buffer.alloc(8 * 1024 ** 2, 42);
|
|
const { script, version } = createScript({ name: 'Promoted plugin', kind: 'plugin', targetRelPath: 'fixture.dll',
|
|
scopeType: 'game', scopeValue: 't5', content });
|
|
db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id);
|
|
const { saveBlueprint, recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
|
|
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { POST } = await import('../../src/routes/api/deployment-collections/+server.js');
|
|
const { load } = await import('../../src/routes/collections/launches/[id]/+page.server.js');
|
|
const { GET: collectionResponse } = await import('../../src/routes/api/deployment-collections/[id]/+server.js');
|
|
const { GET: fleetResponse } = await import('../../src/routes/api/servers/+server.js');
|
|
const { POST: agentHeartbeat } = await import('../../src/routes/api/agent/heartbeat/+server.js');
|
|
const { T5_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
|
|
const blueprint = saveBlueprint({ name: 'Match ten T5 lobbies', entries: T5_ZOMBIE_MAPS.filter(map => map.slug !== 'zombietron')
|
|
.map(map => ({ game: 't5', mapSlug: map.slug, count: 1 })) });
|
|
const preview = previewCollection(blueprint.id, 'exact');
|
|
expect(preview).toMatchObject({ ready: true, mode: 'exact' });
|
|
expect(preview.removals).toHaveLength(9);
|
|
expect(preview.servers).toHaveLength(10);
|
|
const storedPreview = db().prepare('SELECT length(entries) AS bytes FROM server_collection_previews WHERE id=?')
|
|
.get(preview.id) as { bytes: number };
|
|
expect(storedPreview.bytes).toBeLessThan(100_000);
|
|
|
|
const capacity = await import('../../src/lib/server/hosts/capacity.js');
|
|
const placement = vi.spyOn(capacity, 'choosePlacement').mockImplementation(() => {
|
|
throw new Error('Placement is waiting for fresh host capacity');
|
|
});
|
|
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
|
|
const evaluate = preparation.evaluateNativePlacement;
|
|
let placementStarted!: () => void;
|
|
const started = new Promise<void>(resolve => { placementStarted = resolve; });
|
|
const gate = new Promise<void>(resolve => { releasePlacement = resolve; });
|
|
const evaluation = vi.spyOn(preparation, 'evaluateNativePlacement').mockImplementation(async options => {
|
|
placementStarted();
|
|
await gate;
|
|
return evaluate(options);
|
|
});
|
|
const scripts = await import('../../src/lib/server/fleet/scripts.js');
|
|
const fullVersion = vi.spyOn(scripts, 'getVersion');
|
|
|
|
const submit = (keys?: Record<string, string>) => POST({ request: new Request('http://localhost/api/deployment-collections', {
|
|
method: 'POST', body: JSON.stringify({ previewId: preview.id, keys }) }) });
|
|
const response = await submit(Object.fromEntries(preview.servers.map(server => [server.name!, `key-${server.name}`])));
|
|
const body = await response.json();
|
|
expect(response.status).toBe(202);
|
|
expect(body).toMatchObject({ ok: true, collectionId: expect.any(String) });
|
|
expect(await (await submit()).json()).toEqual(body);
|
|
expect(load({ params: { id: body.collectionId } })).toEqual({});
|
|
// Exercise the real route and worker: admission must return before
|
|
// loading launch payloads or inspecting the nine servers to remove.
|
|
expect(nativeRequest).not.toHaveBeenCalled();
|
|
expect(placement).not.toHaveBeenCalled();
|
|
expect(evaluation).not.toHaveBeenCalled();
|
|
expect(fullVersion).not.toHaveBeenCalled();
|
|
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]);
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toHaveLength(9);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(18);
|
|
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toHaveLength(10);
|
|
expect(db().prepare('SELECT id FROM server_launch_previews').all()).toEqual([]);
|
|
const rows = db().prepare('SELECT length(recipe) AS bytes FROM server_launch_recipes').all() as { bytes: number }[];
|
|
expect(rows).toHaveLength(10);
|
|
expect(rows.every(row => row.bytes < 10_000)).toBe(true);
|
|
const collection = getCollection(body.collectionId)!;
|
|
expect(collection.collection).toMatchObject({ mode: 'exact', total: 19 });
|
|
expect(collection.servers.filter(server => server.action === 'destroy')).toHaveLength(9);
|
|
expect(collection.servers.filter(server => server.action === 'create')).toHaveLength(10);
|
|
const { nativeDeployment } = await import('../../src/lib/server/backends/native.js');
|
|
for (const server of collection.servers.filter(server => server.action === 'create')) {
|
|
expect(nativeDeployment(server.id)).toBe(true);
|
|
expect(getDeployment(server.id)?.backend).toBe('native');
|
|
}
|
|
|
|
await new Promise<void>(resolve => setImmediate(resolve));
|
|
await processProvisioning({ wait: false });
|
|
await started;
|
|
expect(evaluation).toHaveBeenCalled();
|
|
expect(placement).not.toHaveBeenCalled();
|
|
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]);
|
|
expect(getCollection(body.collectionId)?.collection.total).toBe(19);
|
|
const collectionPage = collectionResponse({ params: { id: body.collectionId } });
|
|
expect(collectionPage.status).toBe(200);
|
|
expect((await collectionPage.json()).servers).toHaveLength(19);
|
|
const fleet = fleetResponse();
|
|
expect(fleet.status).toBe(200);
|
|
expect((await fleet.json()).servers).toHaveLength(9);
|
|
const heartbeatResponse = await agentHeartbeat({ request: new Request('https://controller.test/api/agent/heartbeat', {
|
|
method: 'POST', headers: { authorization: `Bearer ${agentCredential}` }, body: JSON.stringify({ inventory: heartbeatInventory })
|
|
}) });
|
|
expect(heartbeatResponse.status).toBe(200);
|
|
expect(await heartbeatResponse.json()).toMatchObject({ host: { id: hostId, connected: true }, heartbeatSeconds: 15 });
|
|
// All three endpoints complete while the background placement is still paused.
|
|
expect(placement).not.toHaveBeenCalled();
|
|
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]);
|
|
placement.mockRestore(); fullVersion.mockRestore();
|
|
releasePlacement!();
|
|
await processProvisioning();
|
|
const dispatched = db().prepare(`SELECT n.deployment_id,o.id FROM native_provisioning n
|
|
JOIN ec_host_operations o ON o.id=n.operation_id WHERE o.kind='instance.create'`).all() as { deployment_id: string; id: string }[];
|
|
expect(dispatched).toHaveLength(1);
|
|
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
const payload = getHostOperation(dispatched[0].id)!.payload as { files: { path: string; contentBase64: string; sha256: string }[] };
|
|
expect(payload.files.find(file => file.path.endsWith('/fixture.dll'))).toMatchObject({
|
|
sha256: version.sha256, contentBase64: content.toString('base64')
|
|
});
|
|
|
|
// Queued plans are descriptions, while an attempted launch preserves
|
|
// its exact bytes even after the promoted artifact is deleted.
|
|
setScope({ scriptId: script.id, scopeType: null, scopeValue: null });
|
|
deleteScript(script.id);
|
|
const queued = collection.servers.find(server => server.action === 'create' && server.id !== dispatched[0].deployment_id)!;
|
|
expect(getDeployment(queued.id)?.recipe?.files).toContainEqual(expect.objectContaining({ path: 'fixture.dll', versionId: version.id, size: content.length }));
|
|
const attempted = recipeForDeployment(dispatched[0].deployment_id)!.recipe.files.find(file => file.path === 'fixture.dll');
|
|
expect(attempted).toMatchObject({ sha256: version.sha256, content: content.toString('base64') });
|
|
}, 30_000);
|
|
});
|