import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) })); vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); let fixture: Fixture; let hostId: string; let libraryId: string; const image = 'example.test/plutainer:test'; const releases: (() => void)[] = []; const passes: Promise[] = []; beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture([]); vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password'); const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js'); hostId = createHost({ name: 'Preparation test host', portMin: 28960, portMax: 28980 }).id; const inventory = { cpuCount: 32, cpuUsagePercent: 0, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 1_000_000_000_000, preparedRuntimeImages: [image] }; enrollHost(createEnrollment(hostId).token, inventory); const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); const { db } = await import('../../src/lib/server/db/index.js'); libraryId = createLibraryImport({ hostId, game: 't5' }).id; db() .prepare( "UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?" ) .run('a'.repeat(64), new Date().toISOString(), libraryId); updateHostLibraryCaches(hostId, [ { libraryVersionId: libraryId, status: 'ready', manifestSha256: 'a'.repeat(64) } ]); heartbeat(hostId, { ...inventory, libraries: [{ id: libraryId, state: 'ready' }] }); }); afterEach(async () => { for (const release of releases.splice(0)) release(); await Promise.allSettled(passes.splice(0)); vi.restoreAllMocks(); await fixture.cleanup(); vi.unstubAllEnvs(); }); let queuedKeys = 0; async function queue() { const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); const result = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', hostId, libraryId, image, plutoKey: `saved-key-${++queuedKeys}` }); if (!result.ok) throw new Error(result.message); return result; } async function runPass() { const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); const pass = processNativeProvisioning(() => null); passes.push(pass); await pass; } async function operationFor(deploymentId: string) { const { db } = await import('../../src/lib/server/db/index.js'); return db() .prepare( `SELECT o.id FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=?` ) .get(deploymentId) as { id: string } | undefined; } /** Pause after real bytes are captured so mutations exercise the final admission checks. */ async function pausePreparation() { const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js'); const prepare = preparation.prepareNativeLaunch; let started!: () => void; let release!: () => void; const ready = new Promise((resolve) => { started = resolve; }); const gate = new Promise((resolve) => { release = resolve; }); releases.push(release); const spy = vi .spyOn(preparation, 'prepareNativeLaunch') .mockImplementationOnce(async (input) => { const result = await prepare(input); started(); await gate; return result; }); return { ready, release, spy }; } describe('asynchronous native launch preparation', () => { it('does not place an unsubmitted intent cancelled while host ranking is pending', async () => { const job = await queue(); const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js'); const evaluate = preparation.evaluateNativePlacement; let started!: () => void; let release!: () => void; const ready = new Promise((resolve) => { started = resolve; }); const gate = new Promise((resolve) => { release = resolve; }); releases.push(release); vi.spyOn(preparation, 'evaluateNativePlacement').mockImplementationOnce(async (input) => { const result = await evaluate(input); started(); await gate; return result; }); const pass = runPass(); await ready; const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); expect(nativeInstance(job.name)).toBeNull(); const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String)); release(); await pass; expect(nativeInstance(job.name)).toBeNull(); expect(await operationFor(job.deploymentId)).toBeUndefined(); const { db } = await import('../../src/lib/server/db/index.js'); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]); }); it('keeps controller timers and launch details responsive while preparing multiple large plugins', async () => { const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 }); const { script, version } = createScript({ name: 'Large launch plugin', kind: 'plugin', targetRelPath: 'fixture.dll', scopeType: 'game', scopeValue: 't5', content: Buffer.alloc(8 * 1024 ** 2, 42) }); db() .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') .run(version.id, script.id); const jobs = await Promise.all(Array.from({ length: 4 }, queue)); const hosts = await import('../../src/lib/server/hosts/index.js'); const queueCommand = hosts.queueHostOperationSerialized; let yielded = true; let uninterruptedCommits = 0; const committing = vi .spyOn(hosts, 'queueHostOperationSerialized') .mockImplementation((...args) => { if (!yielded) uninterruptedCommits++; yielded = false; setImmediate(() => { yielded = true; }); return queueCommand(...args); }); const parse = JSON.parse; const stringify = JSON.stringify; const parsing = vi.spyOn(JSON, 'parse').mockImplementation((raw, reviver) => { if (typeof raw === 'string' && raw.length > 1_000_000) throw new Error('Large file payload parsed on the controller thread'); return parse(raw, reviver); }); const encoding = vi .spyOn(JSON, 'stringify') .mockImplementation((...args: Parameters) => { const encoded = stringify(...args); if (encoded && encoded.length > 1_000_000) throw new Error('Large file payload serialized on the controller thread'); return encoded; }); let ticks = 0; const timer = setInterval(() => { ticks++; }, 1); try { const pass = runPass(); await new Promise((resolve) => setTimeout(resolve, 0)); for (const job of jobs) expect(getDeployment(job.deploymentId)?.recipe?.files).toContainEqual( expect.objectContaining({ path: 'fixture.dll', size: 8 * 1024 ** 2 }) ); await pass; expect(ticks).toBeGreaterThan(1); expect(committing).toHaveBeenCalledTimes(4); expect(uninterruptedCommits).toBe(0); for (const job of jobs) { expect(await operationFor(job.deploymentId)).toBeDefined(); expect(getDeployment(job.deploymentId)?.recipe?.files).toContainEqual( expect.objectContaining({ path: 'fixture.dll', sha256: version.sha256 }) ); } } finally { clearInterval(timer); parsing.mockRestore(); encoding.mockRestore(); committing.mockRestore(); } }, 30_000); it('does not resurrect an unsubmitted creation cancelled while its files are prepared', async () => { const job = await queue(); const paused = await pausePreparation(); const pass = runPass(); await paused.ready; const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String)); paused.release(); await pass; const { db } = await import('../../src/lib/server/db/index.js'); expect(await operationFor(job.deploymentId)).toBeUndefined(); expect(db().prepare('SELECT name FROM servers WHERE name=?').get(job.name)).toBeUndefined(); expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]); expect( db().prepare('SELECT state FROM server_deployments WHERE id=?').get(job.deploymentId) ).toEqual({ state: 'cancelled' }); }); it.each(['baseline', 'config', 'script', 'plugin'] as const)( 'discards prepared bytes when the promoted %s changes before dispatch', async (kind) => { const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); const baseline = createScript({ name: 'Launch baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5', content: Buffer.from('set rcon_password "password"\nset sv_voice "1"\n') }); db() .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') .run(baseline.version.id, baseline.script.id); const isConfig = kind === 'baseline' || kind === 'config'; const { script, version } = kind === 'baseline' ? baseline : createScript({ releaseVersion: '1.0.0', name: `Launch ${kind}`, kind, targetRelPath: kind === 'script' ? 'fixture.gsc' : kind === 'plugin' ? 'fixture.dll' : undefined, scopeType: kind === 'config' ? 'map' : 'game', scopeValue: kind === 'config' ? 'zombie_theater' : 't5', content: Buffer.from(isConfig ? 'set sv_voice "1"\n' : 'before promotion') }); db() .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') .run(version.id, script.id); const job = await queue(); const paused = await pausePreparation(); const pass = runPass(); await paused.ready; const content = Buffer.from( isConfig ? 'set rcon_password "password"\nset sv_voice "0"\n' : 'newly promoted file' ); const latest = uploadNewVersion(script.id, content, undefined, '1.0.1'); db() .prepare('UPDATE scripts SET current_version_id=? WHERE id=?') .run(latest.id, script.id); paused.release(); await pass; expect(await operationFor(job.deploymentId)).toBeUndefined(); await runPass(); const operation = await operationFor(job.deploymentId); expect(operation).toBeDefined(); const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); const payload = getHostOperation(operation!.id)!.payload as { files: { contentBase64: string; sha256: string }[]; }; if (isConfig) { const { getPromotedServerConfig } = await import('../../src/lib/server/fleet/server-config.js'); const expected = getPromotedServerConfig( 't5', 'zombie_theater', 'theater', 'test-password' )!; expect(payload.files).toContainEqual( expect.objectContaining({ contentBase64: expected.toString('base64') }) ); expect(expected.toString()).toContain('set sv_voice "0"'); } else expect(payload.files).toContainEqual( expect.objectContaining({ contentBase64: content.toString('base64'), sha256: latest.sha256 }) ); } ); it('discards prepared bytes when the server key changes before dispatch', async () => { const job = await queue(); const paused = await pausePreparation(); const pass = runPass(); await paused.ready; const { db } = await import('../../src/lib/server/db/index.js'); db() .prepare( `UPDATE server_keys SET pluto_key='changed-key' WHERE id IN (SELECT key_id FROM server_key_leases WHERE server_id=? AND is_current=1)` ) .run(job.name); paused.release(); await pass; expect(await operationFor(job.deploymentId)).toBeUndefined(); await runPass(); const operation = await operationFor(job.deploymentId); expect(operation).toBeDefined(); const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); expect(getHostOperation(operation!.id)!.payload).toMatchObject({ env: { PLUTO_SERVER_KEY: 'changed-key' } }); }); it('prepares only one launch while the single startup slot is reserved', async () => { const first = await queue(); const second = await queue(); const paused = await pausePreparation(); const pass = runPass(); await paused.ready; expect(paused.spy).toHaveBeenCalledTimes(1); const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js'); await processNativeProvisioning(() => null, { wait: false }); expect(paused.spy).toHaveBeenCalledTimes(1); paused.release(); await pass; expect(await operationFor(first.deploymentId)).toBeDefined(); expect(await operationFor(second.deploymentId)).toBeUndefined(); }); it('keeps a prepared launch queued while its host is disabled or re-enabling', async () => { const job = await queue(); const paused = await pausePreparation(); const pass = runPass(); await paused.ready; const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(hostId, { mode: 'disabled' }); paused.release(); await pass; expect(await operationFor(job.deploymentId)).toBeUndefined(); expect(updateHost(hostId, { mode: 'active' })).toMatchObject({ mode: 'disabled', readiness: 'preparing' }); await runPass(); expect(await operationFor(job.deploymentId)).toBeUndefined(); }); it('rechecks a reduced startup limit before submitting prepared files', async () => { const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 }); const first = await queue(); await runPass(); expect(await operationFor(first.deploymentId)).toBeDefined(); const second = await queue(); const paused = await pausePreparation(); const pass = runPass(); await paused.ready; saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 }); paused.release(); await pass; expect(await operationFor(second.deploymentId)).toBeUndefined(); saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 }); await runPass(); expect(await operationFor(second.deploymentId)).toBeDefined(); }); });