import { randomUUID } from 'node:crypto'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(async () => 0), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })), rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, status: { players: 0, map: 'zm_transit', gametype: null, roster: [] } })) })); let fixture: Fixture; let hostId: string; let agentCredential: string; let heartbeatInventory: Record; let releasePlacement: (() => void) | undefined; const image = 'example.test/plutainer:test'; beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); releasePlacement = undefined; fixture = await newFixture([]); vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password'); vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); const { createHost, createEnrollment, enrollHost, heartbeat, reserveHostPort } = await import('../../src/lib/server/hosts/index.js'); hostId = createHost({ name: 'Match collection host', portMin: 28960, portMax: 28999 }).id; const inventory = { cpuCount: 64, cpuUsagePercent: 5, memoryTotal: 128 * 1024 ** 3, memoryAvailable: 120 * 1024 ** 3, protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 1_000_000_000_000, runtimeImages: [image] }; agentCredential = enrollHost(createEnrollment(hostId).token, inventory).credential; const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); const { db } = await import('../../src/lib/server/db/index.js'); const libraries = ['t5', 't6'].map(game => { const id = createLibraryImport({ hostId, game }).id; db().prepare("UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?") .run('a'.repeat(64), new Date().toISOString(), id); return { id, state: 'ready' as const }; }); updateHostLibraryCaches(hostId, libraries.map(library => ({ libraryVersionId: library.id, status: 'ready', manifestSha256: 'a'.repeat(64) }))); heartbeatInventory = { ...inventory, libraries }; heartbeat(hostId, heartbeatInventory); const { T6_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js'); const now = new Date().toISOString(); db().transaction(() => { for (const [index, map] of T6_ZOMBIE_MAPS.slice(0, 9).entries()) { const name = `t6_zombie_${map.slug}_1`; const instanceId = randomUUID(); const deploymentId = randomUUID(); const port = 28960 + index; db().prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug,game_port,container_state,last_players,last_seen_at) VALUES(?,'t6',?,'zombie',?,?,'running',0,?)`).run(name, map.canonical, map.slug, port, now); db().prepare('INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,?,?,?,?)') .run(instanceId, name, hostId, libraries[1].id, image, port, now); db().prepare("INSERT INTO server_deployments(id,server,action,state,stage,triggered_by,started_at,finished_at) VALUES(?,?,'create','done','Verified','fixture',?,?)") .run(deploymentId, name, now, now); db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(deploymentId, instanceId); for (const protocol of ['tcp', 'udp'] as const) reserveHostPort(hostId, instanceId, { port, protocol }); } })(); const backend = await import('../../src/lib/server/backends/native.js'); vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => { if (kind !== 'instance.inspect') throw new Error(`Unexpected request ${kind}`); return { exists: true, healthy: true, state: 'running' } as never; }); }); afterEach(async () => { releasePlacement?.(); await new Promise(resolve => setImmediate(resolve)); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); await processProvisioning(); await new Promise(resolve => setImmediate(resolve)); vi.restoreAllMocks(); await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); describe('matching a native collection', () => { it('places queued creations and completes all exact removals during later worker passes', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 10, destroyConcurrency: 9 }); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const { T5_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js'); const blueprint = saveBlueprint({ name: 'Replace nine with ten', entries: T5_ZOMBIE_MAPS .map(map => ({ game: 't5', mapSlug: map.slug, count: 1 })) }); const preview = previewCollection(blueprint.id, 'exact'); const queued = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, `key-${server.name}`])) }); expect(queueCollection({ previewId: preview.id })).toEqual(queued); expect(db().prepare('SELECT id FROM native_instances').all()).toHaveLength(9); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(18); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); await processProvisioning(); const operations = db().prepare("SELECT id,kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.remove')") .all() as { id: string; kind: string }[]; expect(operations.filter(operation => operation.kind === 'instance.create')).toHaveLength(10); const removals = operations.filter(operation => operation.kind === 'instance.remove'); expect(removals).toHaveLength(9); const { reportHostOperationStatus } = await import('../../src/lib/server/hosts/index.js'); for (const operation of removals) reportHostOperationStatus(hostId, operation.id, { state: 'completed', result: { exists: false } }); await processProvisioning(); const active = db().prepare('SELECT id,port FROM native_instances WHERE retired_at IS NULL').all() as { id: string; port: number }[]; expect(active).toHaveLength(10); expect(new Set(active.map(instance => instance.port)).size).toBe(10); for (const instance of active) expect(db().prepare('SELECT protocol,port FROM ec_host_ports WHERE instance_id=? ORDER BY protocol').all(instance.id)) .toEqual([{ protocol: 'tcp', port: instance.port }, { protocol: 'udp', port: instance.port }]); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(20); expect(getCollection(queued.collectionId!)?.servers.filter(server => server.action === 'destroy').map(server => server.state)) .toEqual(Array(9).fill('done')); expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.remove')").all()).toHaveLength(19); }); it('accepts nine removals and ten unplaced creations before capacity checks or file loading', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { createScript, setScope, deleteScript } = await import('../../src/lib/server/fleet/scripts.js'); const content = Buffer.alloc(8 * 1024 ** 2, 42); const { script, version } = createScript({ name: 'Promoted plugin', kind: 'plugin', targetRelPath: 'fixture.dll', scopeType: 'game', scopeValue: 't5', content }); db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id); const { saveBlueprint, recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js'); const { previewCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); const { POST } = await import('../../src/routes/api/deployment-collections/+server.js'); const { load } = await import('../../src/routes/collections/launches/[id]/+page.server.js'); const { GET: collectionResponse } = await import('../../src/routes/api/deployment-collections/[id]/+server.js'); const { GET: fleetResponse } = await import('../../src/routes/api/servers/+server.js'); const { POST: agentHeartbeat } = await import('../../src/routes/api/agent/heartbeat/+server.js'); const { T5_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js'); const blueprint = saveBlueprint({ name: 'Match ten T5 lobbies', entries: T5_ZOMBIE_MAPS .map(map => ({ game: 't5', mapSlug: map.slug, count: 1 })) }); const preview = previewCollection(blueprint.id, 'exact'); expect(preview).toMatchObject({ ready: true, mode: 'exact' }); expect(preview.removals).toHaveLength(9); expect(preview.servers).toHaveLength(10); const storedPreview = db().prepare('SELECT length(entries) AS bytes FROM server_collection_previews WHERE id=?') .get(preview.id) as { bytes: number }; expect(storedPreview.bytes).toBeLessThan(100_000); const capacity = await import('../../src/lib/server/hosts/capacity.js'); const placement = vi.spyOn(capacity, 'choosePlacement').mockImplementation(() => { throw new Error('Placement is waiting for fresh host capacity'); }); const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js'); const evaluate = preparation.evaluateNativePlacement; let placementStarted!: () => void; const started = new Promise(resolve => { placementStarted = resolve; }); const gate = new Promise(resolve => { releasePlacement = resolve; }); const evaluation = vi.spyOn(preparation, 'evaluateNativePlacement').mockImplementation(async options => { placementStarted(); await gate; return evaluate(options); }); const scripts = await import('../../src/lib/server/fleet/scripts.js'); const fullVersion = vi.spyOn(scripts, 'getVersion'); const submit = (keys?: Record) => POST({ request: new Request('http://localhost/api/deployment-collections', { method: 'POST', body: JSON.stringify({ previewId: preview.id, keys }) }) }); const response = await submit(Object.fromEntries(preview.servers.map(server => [server.name!, `key-${server.name}`]))); const body = await response.json(); expect(response.status).toBe(202); expect(body).toMatchObject({ ok: true, collectionId: expect.any(String) }); expect(await (await submit()).json()).toEqual(body); expect(load({ params: { id: body.collectionId } })).toEqual({}); // Exercise the real route and worker: admission must return before // loading launch payloads or inspecting the nine servers to remove. expect(nativeRequest).not.toHaveBeenCalled(); expect(placement).not.toHaveBeenCalled(); expect(evaluation).not.toHaveBeenCalled(); expect(fullVersion).not.toHaveBeenCalled(); expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]); expect(db().prepare('SELECT id FROM native_instances').all()).toHaveLength(9); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(18); expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toHaveLength(10); expect(db().prepare('SELECT id FROM server_launch_previews').all()).toEqual([]); const rows = db().prepare('SELECT length(recipe) AS bytes FROM server_launch_recipes').all() as { bytes: number }[]; expect(rows).toHaveLength(10); expect(rows.every(row => row.bytes < 10_000)).toBe(true); const collection = getCollection(body.collectionId)!; expect(collection.collection).toMatchObject({ mode: 'exact', total: 19 }); expect(collection.servers.filter(server => server.action === 'destroy')).toHaveLength(9); expect(collection.servers.filter(server => server.action === 'create')).toHaveLength(10); const { nativeDeployment } = await import('../../src/lib/server/backends/native.js'); for (const server of collection.servers.filter(server => server.action === 'create')) { expect(nativeDeployment(server.id)).toBe(true); expect(getDeployment(server.id)?.backend).toBe('native'); } await new Promise(resolve => setImmediate(resolve)); await processProvisioning({ wait: false }); await started; expect(evaluation).toHaveBeenCalled(); expect(placement).not.toHaveBeenCalled(); expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]); expect(getCollection(body.collectionId)?.collection.total).toBe(19); const collectionPage = collectionResponse({ params: { id: body.collectionId } }); expect(collectionPage.status).toBe(200); expect((await collectionPage.json()).servers).toHaveLength(19); const fleet = fleetResponse(); expect(fleet.status).toBe(200); expect((await fleet.json()).servers).toHaveLength(9); const heartbeatResponse = await agentHeartbeat({ request: new Request('https://controller.test/api/agent/heartbeat', { method: 'POST', headers: { authorization: `Bearer ${agentCredential}` }, body: JSON.stringify({ inventory: heartbeatInventory }) }) }); expect(heartbeatResponse.status).toBe(200); expect(await heartbeatResponse.json()).toMatchObject({ host: { id: hostId, connected: true }, heartbeatSeconds: 15 }); // All three endpoints complete while the background placement is still paused. expect(placement).not.toHaveBeenCalled(); expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]); placement.mockRestore(); fullVersion.mockRestore(); releasePlacement!(); await processProvisioning(); const dispatched = db().prepare(`SELECT n.deployment_id,o.id FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE o.kind='instance.create'`).all() as { deployment_id: string; id: string }[]; expect(dispatched).toHaveLength(1); const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); const payload = getHostOperation(dispatched[0].id)!.payload as { files: { path: string; contentBase64: string; sha256: string }[] }; expect(payload.files.find(file => file.path.endsWith('/fixture.dll'))).toMatchObject({ sha256: version.sha256, contentBase64: content.toString('base64') }); // Queued plans are descriptions, while an attempted launch preserves // its exact bytes even after the promoted artifact is deleted. setScope({ scriptId: script.id, scopeType: null, scopeValue: null }); deleteScript(script.id); const queued = collection.servers.find(server => server.action === 'create' && server.id !== dispatched[0].deployment_id)!; expect(getDeployment(queued.id)?.recipe?.files).toContainEqual(expect.objectContaining({ path: 'fixture.dll', versionId: version.id, size: content.length })); const attempted = recipeForDeployment(dispatched[0].deployment_id)!.recipe.files.find(file => file.path === 'fixture.dll'); expect(attempted).toMatchObject({ sha256: version.sha256, content: content.toString('base64') }); }, 30_000); });