import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); let fixture: Fixture; let hostId: string; let libraryId: string; const image = 'example.test/plutainer:test'; const releases: (() => void)[] = []; const passes: Promise[] = []; beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password'); vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', ''); const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js'); hostId = createHost({ name: 'Preparation test host', portMin: 28960, portMax: 28980 }).id; const inventory = { cpuCount: 32, cpuUsagePercent: 0, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 1_000_000_000_000, runtimeImages: [image] }; enrollHost(createEnrollment(hostId).token, inventory); const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); const { db } = await import('../../src/lib/server/db/index.js'); libraryId = createLibraryImport({ hostId, game: 't5' }).id; db().prepare("UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?") .run('a'.repeat(64), new Date().toISOString(), libraryId); updateHostLibraryCaches(hostId, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: 'a'.repeat(64) }]); heartbeat(hostId, { ...inventory, libraries: [{ id: libraryId, state: 'ready' }] }); }); afterEach(async () => { for (const release of releases.splice(0)) release(); await Promise.allSettled(passes.splice(0)); vi.restoreAllMocks(); await fixture.cleanup(); vi.unstubAllEnvs(); }); async function queue() { const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); const result = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', hostId, libraryId, image, plutoKey: 'saved-key' }); if (!result.ok) throw new Error(result.message); return result; } async function runPass() { const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); const pass = processNativeProvisioning(() => null); passes.push(pass); await pass; } async function operationFor(deploymentId: string) { const { db } = await import('../../src/lib/server/db/index.js'); return db().prepare(`SELECT o.id FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=?`).get(deploymentId) as { id: string } | undefined; } /** Pause after real bytes are captured so mutations exercise the final admission checks. */ async function pausePreparation() { const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js'); const prepare = preparation.prepareNativeLaunch; let started!: () => void; let release!: () => void; const ready = new Promise(resolve => { started = resolve; }); const gate = new Promise(resolve => { release = resolve; }); releases.push(release); const spy = vi.spyOn(preparation, 'prepareNativeLaunch').mockImplementationOnce(async input => { const result = await prepare(input); started(); await gate; return result; }); return { ready, release, spy }; } describe('asynchronous native launch preparation', () => { it('does not place an unsubmitted intent cancelled while host ranking is pending', async () => { const job = await queue(); const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js'); const evaluate = preparation.evaluateNativePlacement; let started!: () => void; let release!: () => void; const ready = new Promise(resolve => { started = resolve; }); const gate = new Promise(resolve => { release = resolve; }); releases.push(release); vi.spyOn(preparation, 'evaluateNativePlacement').mockImplementationOnce(async input => { const result = await evaluate(input); started(); await gate; return result; }); const pass = runPass(); await ready; const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); expect(nativeInstance(job.name)).toBeNull(); const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String)); release(); await pass; expect(nativeInstance(job.name)).toBeNull(); expect(await operationFor(job.deploymentId)).toBeUndefined(); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]); }); it('keeps controller timers and launch details responsive while preparing multiple large plugins', async () => { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 }); const { script, version } = createScript({ name: 'Large launch plugin', kind: 'plugin', targetRelPath: 'fixture.dll', scopeType: 'game', scopeValue: 't5', content: Buffer.alloc(8 * 1024 ** 2, 42) }); db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id); const jobs = await Promise.all(Array.from({ length: 4 }, queue)); const hosts = await import('../../src/lib/server/hosts/index.js'); const queueCommand = hosts.queueHostOperationSerialized; let yielded = true; let uninterruptedCommits = 0; const committing = vi.spyOn(hosts, 'queueHostOperationSerialized').mockImplementation((...args) => { if (!yielded) uninterruptedCommits++; yielded = false; setImmediate(() => { yielded = true; }); return queueCommand(...args); }); const parse = JSON.parse; const stringify = JSON.stringify; const parsing = vi.spyOn(JSON, 'parse').mockImplementation((raw, reviver) => { if (typeof raw === 'string' && raw.length > 1_000_000) throw new Error('Large file payload parsed on the controller thread'); return parse(raw, reviver); }); const encoding = vi.spyOn(JSON, 'stringify').mockImplementation((...args: Parameters) => { const encoded = stringify(...args); if (encoded && encoded.length > 1_000_000) throw new Error('Large file payload serialized on the controller thread'); return encoded; }); let ticks = 0; const timer = setInterval(() => { ticks++; }, 1); try { const pass = runPass(); await new Promise(resolve => setTimeout(resolve, 0)); for (const job of jobs) expect(getDeployment(job.deploymentId)?.recipe?.files) .toContainEqual(expect.objectContaining({ path: 'fixture.dll', size: 8 * 1024 ** 2 })); await pass; expect(ticks).toBeGreaterThan(1); expect(committing).toHaveBeenCalledTimes(4); expect(uninterruptedCommits).toBe(0); for (const job of jobs) { expect(await operationFor(job.deploymentId)).toBeDefined(); expect(getDeployment(job.deploymentId)?.recipe?.files) .toContainEqual(expect.objectContaining({ path: 'fixture.dll', sha256: version.sha256 })); } } finally { clearInterval(timer); parsing.mockRestore(); encoding.mockRestore(); committing.mockRestore(); } }, 30_000); it('does not resurrect an unsubmitted creation cancelled while its files are prepared', async () => { const job = await queue(); const paused = await pausePreparation(); const pass = runPass(); await paused.ready; const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String)); paused.release(); await pass; const { db } = await import('../../src/lib/server/db/index.js'); expect(await operationFor(job.deploymentId)).toBeUndefined(); expect(db().prepare('SELECT name FROM servers WHERE name=?').get(job.name)).toBeUndefined(); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]); expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(job.deploymentId)).toEqual({ state: 'cancelled' }); }); it('discards prepared bytes when the promoted version changes before dispatch', async () => { const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { script, version } = createScript({ name: 'Launch plugin', kind: 'plugin', targetRelPath: 'fixture.dll', scopeType: 'game', scopeValue: 't5', content: Buffer.from('before promotion') }); db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id); const job = await queue(); const paused = await pausePreparation(); const pass = runPass(); await paused.ready; const content = Buffer.from('newly promoted plugin'); const latest = uploadNewVersion(script.id, content); db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(latest.id, script.id); paused.release(); await pass; expect(await operationFor(job.deploymentId)).toBeUndefined(); await runPass(); const operation = await operationFor(job.deploymentId); expect(operation).toBeDefined(); const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); const payload = getHostOperation(operation!.id)!.payload as { files: { contentBase64: string; sha256: string }[] }; expect(payload.files).toContainEqual(expect.objectContaining({ contentBase64: content.toString('base64'), sha256: latest.sha256 })); }); it('prepares only one launch while the single startup slot is reserved', async () => { const first = await queue(); const second = await queue(); const paused = await pausePreparation(); const pass = runPass(); await paused.ready; expect(paused.spy).toHaveBeenCalledTimes(1); const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); await processNativeProvisioning(() => null, { wait: false }); expect(paused.spy).toHaveBeenCalledTimes(1); paused.release(); await pass; expect(await operationFor(first.deploymentId)).toBeDefined(); expect(await operationFor(second.deploymentId)).toBeUndefined(); }); it('keeps a prepared launch queued if its host is disabled before dispatch', async () => { const job = await queue(); const paused = await pausePreparation(); const pass = runPass(); await paused.ready; const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(hostId, { mode: 'disabled' }); paused.release(); await pass; expect(await operationFor(job.deploymentId)).toBeUndefined(); updateHost(hostId, { mode: 'active' }); await runPass(); expect(await operationFor(job.deploymentId)).toBeDefined(); }); it('rechecks a reduced startup limit before submitting prepared files', async () => { const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 }); const first = await queue(); await runPass(); expect(await operationFor(first.deploymentId)).toBeDefined(); const second = await queue(); const paused = await pausePreparation(); const pass = runPass(); await paused.ready; saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 }); paused.release(); await pass; expect(await operationFor(second.deploymentId)).toBeUndefined(); saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 }); await runPass(); expect(await operationFor(second.deploymentId)).toBeDefined(); }); });