import { createHash } from 'node:crypto'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { join } from 'node:path'; import { newFixture, type Fixture } from '../helpers/fixture.js'; const remote = vi.hoisted(() => ({ state: 'running', players: 0, map: 'zombie_theater' })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: '', status: { players: remote.players, map: remote.map, gametype: 'zom', roster: [] } })), rconGetPlayerCount: vi.fn(async () => remote.players), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })) })); vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); let fixture: Fixture; let hostId: string; let libraryId: string; const image = 'example.test/plutainer:test'; const inventory = () => ({ protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' }] : [] }); beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = ''; fixture = await newFixture([]); vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password'); vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', ''); const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js'); hostId = createHost({ name: 'Native Debian', portMin: 28960, portMax: 28970 }).id; enrollHost(createEnrollment(hostId).token, inventory()); const backend = await import('../../src/lib/server/backends/native.js'); vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => { if (kind !== 'instance.inspect') throw new Error(`Unexpected operation: ${kind}`); return { healthy: remote.state === 'running', state: remote.state, exists: true } as never; }); }); afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); async function prepareLibrary(game: 't5' | 't6' = 't5') { const libs = await import('../../src/lib/server/game-files/index.js'); const version = libs.createLibraryImport({ hostId, game, name: game, version: '1' }); const bytes = Buffer.from('verified base game'); const file = { path: 'zone/base.ff', size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') }; libs.setImportManifest(hostId, version.id, [file]); await libs.uploadImportFile(hostId, version.id, file.path, new Response(bytes).body); const ready = await libs.publishImport(hostId, version.id); libraryId = ready.id; libs.updateHostLibraryCaches(hostId, [{ libraryVersionId: ready.id, status: 'ready', manifestSha256: ready.manifestSha256 }]); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory()); return ready; } async function queue(game: 't5' | 't6' = 't5') { await prepareLibrary(game); remote.map = game === 't5' ? 'zombie_theater' : 'zm_transit'; const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js'); const result = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town', hostId, libraryId, image, plutoKey: 'saved-key' }); if (!result.ok) throw new Error(result.message); return result; } async function operationFor(deploymentId: string) { const { db } = await import('../../src/lib/server/db/index.js'); const row = db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(deploymentId) as { operation_id: string | null }; const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); return row?.operation_id ? getHostOperation(row.operation_id)! : null; } async function advance() { const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); await processNativeProvisioning(() => null); } async function acknowledge(deploymentId: string) { const operation = await operationFor(deploymentId); if (!operation) throw new Error('Missing durable native operation'); const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); reportHostOperation(hostId, operation.id, { state: 'completed', result: { state: remote.state, healthy: remote.state === 'running' } }); await advance(); return operation; } async function detail(id: string) { const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); return getDeployment(id)!.deployment; } describe('native instance provisioning', () => { it.each(['t5', 't6'] as const)('queues and verifies a %s deployment without Komodo', async game => { const job = await queue(game); await advance(); const operation = await operationFor(job.deploymentId); expect(operation).toMatchObject({ kind: 'instance.create', hostId, generation: 1, state: 'pending', payload: { game, libraryId, image, port: job.port } }); expect(operation?.payload.files).toEqual(expect.arrayContaining([expect.objectContaining({ path: 'configs/electric_cherry_map.cfg' })])); await acknowledge(job.deploymentId); expect((await detail(job.deploymentId)).state).toBe('done'); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT protocol FROM ec_host_ports WHERE instance_id=? ORDER BY protocol').all(operation!.instanceId)).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]); }); it('reconciles the same operation after controller restart and an ambiguous remote result', async () => { const job = await queue(); await advance(); const original = (await operationFor(job.deploymentId))!; const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); reportHostOperation(hostId, original.id, { state: 'unknown', error: 'Reply lost after execution' }); const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb(); const { interruptDeployments } = await import('../../src/lib/server/provisioning/history.js'); interruptDeployments(); await advance(); await advance(); expect((await operationFor(job.deploymentId))!.id).toBe(original.id); expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.create'").get()).toEqual({ count: 1 }); expect((await detail(job.deploymentId)).state).toBe('running'); await acknowledge(job.deploymentId); expect((await detail(job.deploymentId)).state).toBe('done'); }); it('holds disconnected instances as unknown without replacement and resumes the original operation', async () => { const job = await queue(); await advance(); const original = (await operationFor(job.deploymentId))!; const { db } = await import('../../src/lib/server/db/index.js'); db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00Z', hostId); await advance(); expect(db().prepare('SELECT container_state FROM servers WHERE name=?').get(job.name)).toEqual({ container_state: 'unknown' }); expect((await operationFor(job.deploymentId))!.id).toBe(original.id); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory()); await acknowledge(job.deploymentId); expect((await detail(job.deploymentId)).state).toBe('done'); }); it('rebuilds only an empty instance and advances its generation after the acknowledged command', async () => { const job = await queue(); await advance(); await acknowledge(job.deploymentId); const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message); remote.players = 2; await advance(); expect(await operationFor(rebuild.deploymentId)).toBeNull(); remote.players = 0; await advance(); const operation = (await operationFor(rebuild.deploymentId))!; expect(operation).toMatchObject({ kind: 'instance.rebuild', generation: 1, payload: { nextGeneration: 2, libraryId } }); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); expect(nativeInstance(job.name)?.generation).toBe(1); await acknowledge(rebuild.deploymentId); expect(nativeInstance(job.name)?.generation).toBe(2); expect((await detail(rebuild.deploymentId)).state).toBe('done'); await advance(); expect(nativeInstance(job.name)?.generation).toBe(2); }); it('blocks new admission on draining hosts and retires existing instances only when empty', async () => { const job = await queue(); await advance(); await acknowledge(job.deploymentId); const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(hostId, { mode: 'draining' }); const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js'); expect(() => nativeSelection({ hostId, libraryId, image }, 't5')).toThrow(/draining/); remote.players = 2; await advance(); const { db } = await import('../../src/lib/server/db/index.js'); const retirement = db().prepare("SELECT id FROM server_deployments WHERE action='destroy' AND server=?").get(job.name) as { id: string }; expect(retirement).toBeTruthy(); expect(await operationFor(retirement.id)).toBeNull(); remote.players = 0; await advance(); const remove = await acknowledge(retirement.id); expect(remove.kind).toBe('instance.remove'); expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined(); expect(db().prepare('SELECT * FROM ec_host_ports WHERE instance_id=?').all(remove.instanceId)).toEqual([]); const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js'); expect(getLibraryVersion(libraryId)?.state).toBe('ready'); }); it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => { const job = await queue(); await advance(); await acknowledge(job.deploymentId); const { db } = await import('../../src/lib/server/db/index.js'); const before = Buffer.from('original bytes on the host'); const after = Buffer.from('new campaign bytes'); db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote',?)").run(new Date().toISOString()); db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign',?,1,'pending')").run(job.name); const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after) VALUES('campaign',?,'updated','hud.gsc',?)").run(job.name, after).lastInsertRowid); const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js'); queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign', server: job.name, kind: 'script', relPath: 'hud.gsc', content: after, inventorySource: 'test', deploymentFileId: fileId }); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); const restart = requestRestart(job.name, { kind: 'campaign', id: 'campaign' }); remote.players = 2; await advance(); const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string }; expect(staged.stage_operation_id).toBeTruthy(); const { getHostOperation, reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); const stage = getHostOperation(staged.stage_operation_id)!; expect(stage.kind).toBe('files.stage'); expect(await operationFor(restart.id)).toBeNull(); expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('pending'); reportHostOperation(hostId, stage.id, { state: 'completed', result: { state: 'staged' } }); await advance(); expect(await operationFor(restart.id)).toBeNull(); remote.players = 0; await advance(); const activation = (await operationFor(restart.id))!; expect(activation).toMatchObject({ kind: 'instance.restart', generation: 1, payload: { stageId: restart.id } }); expect(db().prepare('SELECT content_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null }); const files = stage.payload.files as Array<{ id: number; path: string }>; reportHostOperation(hostId, activation.id, { state: 'completed', result: { before: [{ id: files[0].id, path: files[0].path, contentBase64: before.toString('base64') }] } }); await advance(); expect((await detail(restart.id)).state).toBe('done'); expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('applied'); expect(db().prepare('SELECT content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex') }); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe(createHash('sha256').update(after).digest('hex')); }); it('atomically launches mixed-game collections with per-host port reservations', async () => { const t5 = await prepareLibrary('t5'); const t6 = await prepareLibrary('t6'); const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); const blueprint = saveBlueprint({ name: 'Both games', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }, { game: 't6', mapSlug: 'town', count: 1 }] }); const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); const preview = previewCollection(blueprint.id, 'fill', { hostId, libraryIds: { t5: t5.id, t6: t6.id }, image }); expect(preview.ready).toBe(true); expect(preview.servers.map(server => server.port)).toEqual([28960, 28961, 28962]); const queued = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }); expect(queueCollection({ previewId: preview.id }).collectionId).toBe(queued.collectionId); const { db } = await import('../../src/lib/server/db/index.js'); const instances = db().prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port').all(); expect(instances).toEqual([{ host_id: hostId, library_id: t5.id, port: 28960 }, { host_id: hostId, library_id: t5.id, port: 28961 }, { host_id: hostId, library_id: t6.id, port: 28962 }]); const { createHost, createEnrollment, enrollHost, hostPortAvailable } = await import('../../src/lib/server/hosts/index.js'); const secondHost = createHost({ name: 'Second Debian' }); enrollHost(createEnrollment(secondHost.id).token, inventory()); const { updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); updateHostLibraryCaches(secondHost.id, [{ libraryVersionId: t5.id, status: 'ready', manifestSha256: t5.manifestSha256 }]); const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js'); const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'moon', hostId: secondHost.id, libraryId: t5.id, image, plutoKey: 'second-host-key' }); expect(second.ok).toBe(true); if (!second.ok) throw new Error(second.message); expect(second.port).toBe(28960); expect(hostPortAvailable(hostId, 28960)).toBe(false); expect(hostPortAvailable(secondHost.id, 28960)).toBe(false); expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ count: 8 }); }); it('reads remote instance files through the agent including missing files and epoch timestamps', async () => { const job = await queue(); const backend = await import('../../src/lib/server/backends/native.js'); const remoteRead = vi.mocked(backend.nativeRequest); remoteRead.mockImplementation(async (_instance, kind, payload) => { if (payload?.path === 'missing') return (kind === 'files.read' ? { contentBase64: null } : null) as never; if (kind === 'files.stat') return { size: 15, mtime: 1_790_000_000, isFile: true, isDirectory: false } as never; if (kind === 'files.read') return { contentBase64: Buffer.from('agent file bytes').toString('base64') } as never; if (kind === 'files.list') return { entries: [{ name: 'configs', isDirectory: true, isFile: false }] } as never; throw new Error('Unexpected file operation'); }); const files = await import('../../src/lib/server/backends/files.js'); const root = join(fixture.t5, job.name); expect((await files.readFile(join(root, 'configs/server.cfg'))).toString()).toBe('agent file bytes'); const info = await files.stat(join(root, 'configs/server.cfg')); expect(info.mtime.getTime()).toBe(1_790_000_000_000); expect(info.isFile()).toBe(true); expect(await files.exists(join(root, 'missing'))).toBe(false); await expect(files.readFile(join(root, 'missing'))).rejects.toMatchObject({ code: 'ENOENT' }); const entries = await files.readdir(root, { withFileTypes: true }); expect(entries[0].name).toBe('configs'); expect(entries[0].isDirectory()).toBe(true); expect(remoteRead).toHaveBeenCalledWith(expect.objectContaining({ server: job.name }), 'files.read', { path: 'configs/server.cfg' }); const { stat } = await import('node:fs/promises'); await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' }); }); it('retires an agent-confirmed missing container without waiting on nonexistent RCON', async () => { const job = await queue(); await advance(); await acknowledge(job.deploymentId); const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); const removal = destroyServer(job.name); if (!removal.ok || !removal.deploymentId) throw new Error('Removal was not queued'); const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconStatusResult).mockClear(); remote.state = 'missing'; await advance(); expect((await operationFor(removal.deploymentId))?.kind).toBe('instance.remove'); expect(rconStatusResult).not.toHaveBeenCalled(); await acknowledge(removal.deploymentId); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); expect(nativeInstance(job.name)).toBeNull(); expect((await detail(removal.deploymentId)).state).toBe('done'); }); it('releases a failed activation slot without recording fictitious file changes', async () => { const job = await queue(); await advance(); await acknowledge(job.deploymentId); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('preflight-failed','promote',?)").run(new Date().toISOString()); db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('preflight-failed',?,1,'pending')").run(job.name); const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path) VALUES('preflight-failed',?,'updated','hud.gsc')").run(job.name).lastInsertRowid); const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js'); queueFileChange({ sourceKind: 'campaign', sourceId: 'preflight-failed', server: job.name, kind: 'script', relPath: 'hud.gsc', content: Buffer.from('new bytes'), inventorySource: 'test', deploymentFileId: fileId }); const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); const restart = requestRestart(job.name, { kind: 'campaign', id: 'preflight-failed' }); await advance(); const stage = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string }; const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); reportHostOperation(hostId, stage.stage_operation_id, { state: 'completed', result: { state: 'staged' } }); await advance(); const activation = (await operationFor(restart.id))!; expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('applying'); reportHostOperation(hostId, activation.id, { state: 'failed', error: 'Target changed after campaign review; no activation occurred' }); await advance(); expect((await detail(restart.id)).state).toBe('failed'); expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('pending'); expect(db().prepare('SELECT COUNT(*) AS count FROM restart_rollout WHERE healthy_at IS NULL').get()).toEqual({ count: 0 }); expect(db().prepare('SELECT content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, sha256_before: null }); const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js'); expect(getServerScript(job.name, 'hud.gsc')).toBeUndefined(); }); it('cancels a queued native launch locally and releases both ports without submitting a create', async () => { const job = await queue(); const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); const instance = nativeInstance(job.name)!; const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); const cancelled = destroyServer(job.name); expect(cancelled.ok).toBe(true); await advance(); const { db } = await import('../../src/lib/server/db/index.js'); expect((await detail(job.deploymentId)).state).toBe('cancelled'); expect(nativeInstance(job.name)).toBeNull(); expect(db().prepare('SELECT * FROM ec_host_ports WHERE instance_id=?').all(instance.id)).toEqual([]); expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind LIKE 'instance.%'").get()).toEqual({ count: 0 }); const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js'); expect(getLibraryVersion(libraryId)?.state).toBe('ready'); const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); expect(getServerKey(job.name)).toBe('saved-key'); }); });