391 lines
18 KiB
TypeScript
391 lines
18 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
|
rconGetPlayerCount: vi.fn(async () => 0),
|
|
rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })),
|
|
rconStatusResult: vi.fn(async () => ({
|
|
ok: true,
|
|
endpoint: { host: 'agent', port: 28960 },
|
|
status: { players: 0, map: 'zm_transit', gametype: null, roster: [] }
|
|
}))
|
|
}));
|
|
|
|
let fixture: Fixture;
|
|
let hostId: string;
|
|
let agentCredential: string;
|
|
let heartbeatInventory: Record<string, unknown>;
|
|
let releasePlacement: (() => void) | undefined;
|
|
const image = 'example.test/plutainer:test';
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.clearAllMocks();
|
|
releasePlacement = undefined;
|
|
fixture = await newFixture([]);
|
|
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
|
|
vi.stubEnv('PLUTAINER_IMAGE', image);
|
|
vi.stubEnv('RCON_PASSWORD', 'test-password');
|
|
|
|
vi.stubGlobal(
|
|
'fetch',
|
|
vi.fn(async () =>
|
|
Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })
|
|
)
|
|
);
|
|
const { createHost, createEnrollment, enrollHost, heartbeat, reserveHostPort } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
hostId = createHost({ name: 'Match collection host', portMin: 28960, portMax: 28999 }).id;
|
|
const inventory = {
|
|
cpuCount: 64,
|
|
cpuUsagePercent: 5,
|
|
memoryTotal: 128 * 1024 ** 3,
|
|
memoryAvailable: 120 * 1024 ** 3,
|
|
protocolVersion: 1,
|
|
dockerRunning: true,
|
|
directoriesWritable: true,
|
|
storageAvailable: 1_000_000_000_000,
|
|
preparedRuntimeImages: [image]
|
|
};
|
|
agentCredential = enrollHost(createEnrollment(hostId).token, inventory).credential;
|
|
const { createLibraryImport, updateHostLibraryCaches } =
|
|
await import('../../src/lib/server/game-files/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const libraries = ['t5', 't6'].map((game) => {
|
|
const id = createLibraryImport({ hostId, game }).id;
|
|
db()
|
|
.prepare(
|
|
"UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?"
|
|
)
|
|
.run('a'.repeat(64), new Date().toISOString(), id);
|
|
return { id, state: 'ready' as const };
|
|
});
|
|
updateHostLibraryCaches(
|
|
hostId,
|
|
libraries.map((library) => ({
|
|
libraryVersionId: library.id,
|
|
status: 'ready',
|
|
manifestSha256: 'a'.repeat(64)
|
|
}))
|
|
);
|
|
heartbeatInventory = { ...inventory, libraries };
|
|
heartbeat(hostId, heartbeatInventory);
|
|
const { T6_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
|
|
const now = new Date().toISOString();
|
|
db().transaction(() => {
|
|
for (const [index, map] of T6_ZOMBIE_MAPS.slice(0, 9).entries()) {
|
|
const name = `t6_zombie_${map.slug}_1`;
|
|
const instanceId = randomUUID();
|
|
const deploymentId = randomUUID();
|
|
const port = 28960 + index;
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO servers(name,game,map,game_mode,map_slug,game_port,container_state,last_players,last_seen_at)
|
|
VALUES(?,'t6',?,'zombie',?,?,'running',0,?)`
|
|
)
|
|
.run(name, map.canonical, map.slug, port, now);
|
|
db()
|
|
.prepare(
|
|
'INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,?,?,?,?)'
|
|
)
|
|
.run(instanceId, name, hostId, libraries[1].id, image, port, now);
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO server_deployments(id,server,action,state,stage,triggered_by,started_at,finished_at) VALUES(?,?,'create','done','Verified','fixture',?,?)"
|
|
)
|
|
.run(deploymentId, name, now, now);
|
|
db()
|
|
.prepare(
|
|
'INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)'
|
|
)
|
|
.run(deploymentId, instanceId);
|
|
for (const protocol of ['tcp', 'udp'] as const)
|
|
reserveHostPort(hostId, instanceId, { port, protocol });
|
|
}
|
|
})();
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => {
|
|
if (kind !== 'instance.inspect') throw new Error(`Unexpected request ${kind}`);
|
|
return { exists: true, healthy: true, state: 'running' } as never;
|
|
});
|
|
});
|
|
|
|
afterEach(async () => {
|
|
releasePlacement?.();
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
vi.restoreAllMocks();
|
|
await fixture.cleanup();
|
|
vi.unstubAllEnvs();
|
|
vi.unstubAllGlobals();
|
|
});
|
|
|
|
describe('matching a native collection', () => {
|
|
it('places queued creations and completes all exact removals during later worker passes', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { fleetSettings, saveFleetSettings } =
|
|
await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 10, destroyConcurrency: 9 });
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, queueCollection, getCollection } =
|
|
await import('../../src/lib/server/fleet/collections.js');
|
|
const { T5_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
|
|
const blueprint = saveBlueprint({
|
|
name: 'Replace nine with ten',
|
|
entries: T5_ZOMBIE_MAPS.map((map) => ({ game: 't5', mapSlug: map.slug, count: 1 }))
|
|
});
|
|
const preview = previewCollection(blueprint.id, 'exact');
|
|
const queued = queueCollection({
|
|
previewId: preview.id,
|
|
keys: Object.fromEntries(
|
|
preview.servers.map((server) => [server.name!, `key-${server.name}`])
|
|
)
|
|
});
|
|
expect(queueCollection({ previewId: preview.id })).toEqual(queued);
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toHaveLength(9);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(18);
|
|
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning();
|
|
const operations = db()
|
|
.prepare(
|
|
"SELECT id,kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.remove')"
|
|
)
|
|
.all() as { id: string; kind: string }[];
|
|
expect(operations.filter((operation) => operation.kind === 'instance.create')).toHaveLength(
|
|
10
|
|
);
|
|
const removals = operations.filter((operation) => operation.kind === 'instance.remove');
|
|
expect(removals).toHaveLength(9);
|
|
const { reportHostOperationStatus } = await import('../../src/lib/server/hosts/index.js');
|
|
for (const operation of removals)
|
|
reportHostOperationStatus(hostId, operation.id, {
|
|
state: 'completed',
|
|
result: { exists: false }
|
|
});
|
|
await processProvisioning();
|
|
const active = db()
|
|
.prepare('SELECT id,port FROM native_instances WHERE retired_at IS NULL')
|
|
.all() as { id: string; port: number }[];
|
|
expect(active).toHaveLength(10);
|
|
expect(new Set(active.map((instance) => instance.port)).size).toBe(10);
|
|
for (const instance of active)
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
'SELECT protocol,port FROM ec_host_ports WHERE instance_id=? ORDER BY protocol'
|
|
)
|
|
.all(instance.id)
|
|
).toEqual([
|
|
{ protocol: 'tcp', port: instance.port },
|
|
{ protocol: 'udp', port: instance.port }
|
|
]);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(20);
|
|
expect(
|
|
getCollection(queued.collectionId!)
|
|
?.servers.filter((server) => server.action === 'destroy')
|
|
.map((server) => server.state)
|
|
).toEqual(Array(9).fill('done'));
|
|
expect(
|
|
db()
|
|
.prepare(
|
|
"SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.remove')"
|
|
)
|
|
.all()
|
|
).toHaveLength(19);
|
|
});
|
|
|
|
it('accepts nine removals and ten unplaced creations before capacity checks or file loading', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { createScript, setScope, deleteScript } =
|
|
await import('../../src/lib/server/fleet/scripts.js');
|
|
const content = Buffer.alloc(8 * 1024 ** 2, 42);
|
|
const { script, version } = createScript({
|
|
name: 'Promoted plugin',
|
|
kind: 'plugin',
|
|
targetRelPath: 'fixture.dll',
|
|
scopeType: 'game',
|
|
scopeValue: 't5',
|
|
content
|
|
});
|
|
db()
|
|
.prepare('UPDATE scripts SET current_version_id=? WHERE id=?')
|
|
.run(version.id, script.id);
|
|
const { saveBlueprint, recipeForDeployment } =
|
|
await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, getCollection } =
|
|
await import('../../src/lib/server/fleet/collections.js');
|
|
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
const { POST } = await import('../../src/routes/api/deployment-collections/+server.js');
|
|
const { load } = await import('../../src/routes/collections/launches/[id]/+page.server.js');
|
|
const { GET: collectionResponse } =
|
|
await import('../../src/routes/api/deployment-collections/[id]/+server.js');
|
|
const { GET: fleetResponse } = await import('../../src/routes/api/servers/+server.js');
|
|
const { POST: agentHeartbeat } =
|
|
await import('../../src/routes/api/agent/heartbeat/+server.js');
|
|
const { T5_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
|
|
const blueprint = saveBlueprint({
|
|
name: 'Match ten T5 lobbies',
|
|
entries: T5_ZOMBIE_MAPS.map((map) => ({ game: 't5', mapSlug: map.slug, count: 1 }))
|
|
});
|
|
const preview = previewCollection(blueprint.id, 'exact');
|
|
expect(preview).toMatchObject({ ready: true, mode: 'exact' });
|
|
expect(preview.removals).toHaveLength(9);
|
|
expect(preview.servers).toHaveLength(10);
|
|
const storedPreview = db()
|
|
.prepare('SELECT length(entries) AS bytes FROM server_collection_previews WHERE id=?')
|
|
.get(preview.id) as { bytes: number };
|
|
expect(storedPreview.bytes).toBeLessThan(100_000);
|
|
|
|
const capacity = await import('../../src/lib/server/hosts/capacity.js');
|
|
const placement = vi.spyOn(capacity, 'choosePlacement').mockImplementation(() => {
|
|
throw new Error('Placement is waiting for fresh host capacity');
|
|
});
|
|
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
|
|
const evaluate = preparation.evaluateNativePlacement;
|
|
let placementStarted!: () => void;
|
|
const started = new Promise<void>((resolve) => {
|
|
placementStarted = resolve;
|
|
});
|
|
const gate = new Promise<void>((resolve) => {
|
|
releasePlacement = resolve;
|
|
});
|
|
const evaluation = vi
|
|
.spyOn(preparation, 'evaluateNativePlacement')
|
|
.mockImplementation(async (options) => {
|
|
placementStarted();
|
|
await gate;
|
|
return evaluate(options);
|
|
});
|
|
const scripts = await import('../../src/lib/server/fleet/scripts.js');
|
|
const fullVersion = vi.spyOn(scripts, 'getVersion');
|
|
|
|
const submit = (keys?: Record<string, string>) =>
|
|
POST({
|
|
request: new Request('http://localhost/api/deployment-collections', {
|
|
method: 'POST',
|
|
body: JSON.stringify({ previewId: preview.id, keys })
|
|
})
|
|
});
|
|
const response = await submit(
|
|
Object.fromEntries(
|
|
preview.servers.map((server) => [server.name!, `key-${server.name}`])
|
|
)
|
|
);
|
|
const body = await response.json();
|
|
expect(response.status).toBe(202);
|
|
expect(body).toMatchObject({ ok: true, collectionId: expect.any(String) });
|
|
expect(await (await submit()).json()).toEqual(body);
|
|
expect(load({ params: { id: body.collectionId } })).toEqual({});
|
|
// Exercise the real route and worker: admission must return before
|
|
// loading launch payloads or inspecting the nine servers to remove.
|
|
expect(nativeRequest).not.toHaveBeenCalled();
|
|
expect(placement).not.toHaveBeenCalled();
|
|
expect(evaluation).not.toHaveBeenCalled();
|
|
expect(fullVersion).not.toHaveBeenCalled();
|
|
expect(
|
|
db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()
|
|
).toEqual([]);
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toHaveLength(9);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(18);
|
|
expect(
|
|
db().prepare('SELECT deployment_id FROM native_creation_intents').all()
|
|
).toHaveLength(10);
|
|
expect(db().prepare('SELECT id FROM server_launch_previews').all()).toEqual([]);
|
|
const rows = db()
|
|
.prepare('SELECT length(recipe) AS bytes FROM server_launch_recipes')
|
|
.all() as { bytes: number }[];
|
|
expect(rows).toHaveLength(10);
|
|
expect(rows.every((row) => row.bytes < 10_000)).toBe(true);
|
|
const collection = getCollection(body.collectionId)!;
|
|
expect(collection.collection).toMatchObject({ mode: 'exact', total: 19 });
|
|
expect(collection.servers.filter((server) => server.action === 'destroy')).toHaveLength(9);
|
|
expect(collection.servers.filter((server) => server.action === 'create')).toHaveLength(10);
|
|
const { nativeDeployment } = await import('../../src/lib/server/backends/native.js');
|
|
for (const server of collection.servers.filter((server) => server.action === 'create')) {
|
|
expect(nativeDeployment(server.id)).toBe(true);
|
|
expect(getDeployment(server.id)?.backend).toBe('native');
|
|
}
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
await processProvisioning({ wait: false });
|
|
await started;
|
|
expect(evaluation).toHaveBeenCalled();
|
|
expect(placement).not.toHaveBeenCalled();
|
|
expect(
|
|
db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()
|
|
).toEqual([]);
|
|
expect(getCollection(body.collectionId)?.collection.total).toBe(19);
|
|
const collectionPage = collectionResponse({ params: { id: body.collectionId } });
|
|
expect(collectionPage.status).toBe(200);
|
|
expect((await collectionPage.json()).servers).toHaveLength(19);
|
|
const fleet = fleetResponse();
|
|
expect(fleet.status).toBe(200);
|
|
expect((await fleet.json()).servers).toHaveLength(9);
|
|
const heartbeatResponse = await agentHeartbeat({
|
|
request: new Request('https://controller.test/api/agent/heartbeat', {
|
|
method: 'POST',
|
|
headers: { authorization: `Bearer ${agentCredential}` },
|
|
body: JSON.stringify({ inventory: heartbeatInventory })
|
|
})
|
|
});
|
|
expect(heartbeatResponse.status).toBe(200);
|
|
expect(await heartbeatResponse.json()).toMatchObject({
|
|
host: { id: hostId, connected: true },
|
|
heartbeatSeconds: 15
|
|
});
|
|
// All three endpoints complete while the background placement is still paused.
|
|
expect(placement).not.toHaveBeenCalled();
|
|
expect(
|
|
db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()
|
|
).toEqual([]);
|
|
placement.mockRestore();
|
|
fullVersion.mockRestore();
|
|
releasePlacement!();
|
|
await processProvisioning();
|
|
const dispatched = db()
|
|
.prepare(
|
|
`SELECT n.deployment_id,o.id FROM native_provisioning n
|
|
JOIN ec_host_operations o ON o.id=n.operation_id WHERE o.kind='instance.create'`
|
|
)
|
|
.all() as { deployment_id: string; id: string }[];
|
|
expect(dispatched).toHaveLength(1);
|
|
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
const payload = getHostOperation(dispatched[0].id)!.payload as {
|
|
files: { path: string; contentBase64: string; sha256: string }[];
|
|
};
|
|
expect(payload.files.find((file) => file.path.endsWith('/fixture.dll'))).toMatchObject({
|
|
sha256: version.sha256,
|
|
contentBase64: content.toString('base64')
|
|
});
|
|
|
|
// Queued plans are descriptions, while an attempted launch preserves
|
|
// its exact bytes even after the promoted artifact is deleted.
|
|
setScope({ scriptId: script.id, scopeType: null, scopeValue: null });
|
|
deleteScript(script.id);
|
|
const queued = collection.servers.find(
|
|
(server) => server.action === 'create' && server.id !== dispatched[0].deployment_id
|
|
)!;
|
|
expect(getDeployment(queued.id)?.recipe?.files).toContainEqual(
|
|
expect.objectContaining({
|
|
path: 'fixture.dll',
|
|
versionId: version.id,
|
|
size: content.length
|
|
})
|
|
);
|
|
const attempted = recipeForDeployment(dispatched[0].deployment_id)!.recipe.files.find(
|
|
(file) => file.path === 'fixture.dll'
|
|
);
|
|
expect(attempted).toMatchObject({
|
|
sha256: version.sha256,
|
|
content: content.toString('base64')
|
|
});
|
|
}, 30_000);
|
|
});
|