256 lines
13 KiB
TypeScript
256 lines
13 KiB
TypeScript
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({
|
|
checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354)
|
|
}));
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
|
|
|
|
let fixture: Fixture;
|
|
let hostId: string;
|
|
let libraryId: string;
|
|
const image = 'example.test/plutainer:test';
|
|
const releases: (() => void)[] = [];
|
|
const passes: Promise<void>[] = [];
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules(); vi.clearAllMocks();
|
|
fixture = await newFixture([]);
|
|
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
|
|
vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password');
|
|
|
|
const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
hostId = createHost({ name: 'Preparation test host', portMin: 28960, portMax: 28980 }).id;
|
|
const inventory = { cpuCount: 32, cpuUsagePercent: 0, memoryTotal: 64 * 1024 ** 3,
|
|
memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true,
|
|
directoriesWritable: true, storageAvailable: 1_000_000_000_000, runtimeImages: [image] };
|
|
enrollHost(createEnrollment(hostId).token, inventory);
|
|
const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
libraryId = createLibraryImport({ hostId, game: 't5' }).id;
|
|
db().prepare("UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?")
|
|
.run('a'.repeat(64), new Date().toISOString(), libraryId);
|
|
updateHostLibraryCaches(hostId, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: 'a'.repeat(64) }]);
|
|
heartbeat(hostId, { ...inventory, libraries: [{ id: libraryId, state: 'ready' }] });
|
|
});
|
|
|
|
afterEach(async () => {
|
|
for (const release of releases.splice(0)) release();
|
|
await Promise.allSettled(passes.splice(0));
|
|
vi.restoreAllMocks();
|
|
await fixture.cleanup();
|
|
vi.unstubAllEnvs();
|
|
});
|
|
|
|
let queuedKeys = 0;
|
|
async function queue() {
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const result = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater',
|
|
hostId, libraryId, image, plutoKey: `saved-key-${++queuedKeys}` });
|
|
if (!result.ok) throw new Error(result.message);
|
|
return result;
|
|
}
|
|
|
|
async function runPass() {
|
|
const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
|
|
const pass = processNativeProvisioning(() => null);
|
|
passes.push(pass);
|
|
await pass;
|
|
}
|
|
|
|
async function operationFor(deploymentId: string) {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
return db().prepare(`SELECT o.id FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id
|
|
WHERE n.deployment_id=?`).get(deploymentId) as { id: string } | undefined;
|
|
}
|
|
|
|
/** Pause after real bytes are captured so mutations exercise the final admission checks. */
|
|
async function pausePreparation() {
|
|
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
|
|
const prepare = preparation.prepareNativeLaunch;
|
|
let started!: () => void;
|
|
let release!: () => void;
|
|
const ready = new Promise<void>(resolve => { started = resolve; });
|
|
const gate = new Promise<void>(resolve => { release = resolve; });
|
|
releases.push(release);
|
|
const spy = vi.spyOn(preparation, 'prepareNativeLaunch').mockImplementationOnce(async input => {
|
|
const result = await prepare(input);
|
|
started();
|
|
await gate;
|
|
return result;
|
|
});
|
|
return { ready, release, spy };
|
|
}
|
|
|
|
describe('asynchronous native launch preparation', () => {
|
|
it('does not place an unsubmitted intent cancelled while host ranking is pending', async () => {
|
|
const job = await queue();
|
|
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
|
|
const evaluate = preparation.evaluateNativePlacement;
|
|
let started!: () => void;
|
|
let release!: () => void;
|
|
const ready = new Promise<void>(resolve => { started = resolve; });
|
|
const gate = new Promise<void>(resolve => { release = resolve; });
|
|
releases.push(release);
|
|
vi.spyOn(preparation, 'evaluateNativePlacement').mockImplementationOnce(async input => {
|
|
const result = await evaluate(input);
|
|
started(); await gate;
|
|
return result;
|
|
});
|
|
const pass = runPass();
|
|
await ready;
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)).toBeNull();
|
|
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String));
|
|
release(); await pass;
|
|
expect(nativeInstance(job.name)).toBeNull();
|
|
expect(await operationFor(job.deploymentId)).toBeUndefined();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
});
|
|
|
|
it('keeps controller timers and launch details responsive while preparing multiple large plugins', async () => {
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 });
|
|
const { script, version } = createScript({ name: 'Large launch plugin', kind: 'plugin', targetRelPath: 'fixture.dll',
|
|
scopeType: 'game', scopeValue: 't5', content: Buffer.alloc(8 * 1024 ** 2, 42) });
|
|
db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id);
|
|
const jobs = await Promise.all(Array.from({ length: 4 }, queue));
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const queueCommand = hosts.queueHostOperationSerialized;
|
|
let yielded = true;
|
|
let uninterruptedCommits = 0;
|
|
const committing = vi.spyOn(hosts, 'queueHostOperationSerialized').mockImplementation((...args) => {
|
|
if (!yielded) uninterruptedCommits++;
|
|
yielded = false;
|
|
setImmediate(() => { yielded = true; });
|
|
return queueCommand(...args);
|
|
});
|
|
const parse = JSON.parse;
|
|
const stringify = JSON.stringify;
|
|
const parsing = vi.spyOn(JSON, 'parse').mockImplementation((raw, reviver) => {
|
|
if (typeof raw === 'string' && raw.length > 1_000_000) throw new Error('Large file payload parsed on the controller thread');
|
|
return parse(raw, reviver);
|
|
});
|
|
const encoding = vi.spyOn(JSON, 'stringify').mockImplementation((...args: Parameters<typeof JSON.stringify>) => {
|
|
const encoded = stringify(...args);
|
|
if (encoded && encoded.length > 1_000_000) throw new Error('Large file payload serialized on the controller thread');
|
|
return encoded;
|
|
});
|
|
let ticks = 0;
|
|
const timer = setInterval(() => { ticks++; }, 1);
|
|
try {
|
|
const pass = runPass();
|
|
await new Promise<void>(resolve => setTimeout(resolve, 0));
|
|
for (const job of jobs) expect(getDeployment(job.deploymentId)?.recipe?.files)
|
|
.toContainEqual(expect.objectContaining({ path: 'fixture.dll', size: 8 * 1024 ** 2 }));
|
|
await pass;
|
|
expect(ticks).toBeGreaterThan(1);
|
|
expect(committing).toHaveBeenCalledTimes(4);
|
|
expect(uninterruptedCommits).toBe(0);
|
|
for (const job of jobs) {
|
|
expect(await operationFor(job.deploymentId)).toBeDefined();
|
|
expect(getDeployment(job.deploymentId)?.recipe?.files)
|
|
.toContainEqual(expect.objectContaining({ path: 'fixture.dll', sha256: version.sha256 }));
|
|
}
|
|
} finally {
|
|
clearInterval(timer); parsing.mockRestore(); encoding.mockRestore(); committing.mockRestore();
|
|
}
|
|
}, 30_000);
|
|
|
|
it('does not resurrect an unsubmitted creation cancelled while its files are prepared', async () => {
|
|
const job = await queue();
|
|
const paused = await pausePreparation();
|
|
const pass = runPass();
|
|
await paused.ready;
|
|
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String));
|
|
paused.release();
|
|
await pass;
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(await operationFor(job.deploymentId)).toBeUndefined();
|
|
expect(db().prepare('SELECT name FROM servers WHERE name=?').get(job.name)).toBeUndefined();
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(job.deploymentId)).toEqual({ state: 'cancelled' });
|
|
});
|
|
|
|
it('discards prepared bytes when the promoted version changes before dispatch', async () => {
|
|
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { script, version } = createScript({ name: 'Launch plugin', kind: 'plugin', targetRelPath: 'fixture.dll',
|
|
scopeType: 'game', scopeValue: 't5', content: Buffer.from('before promotion') });
|
|
db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id);
|
|
const job = await queue();
|
|
const paused = await pausePreparation();
|
|
const pass = runPass();
|
|
await paused.ready;
|
|
const content = Buffer.from('newly promoted plugin');
|
|
const latest = uploadNewVersion(script.id, content);
|
|
db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(latest.id, script.id);
|
|
paused.release();
|
|
await pass;
|
|
expect(await operationFor(job.deploymentId)).toBeUndefined();
|
|
await runPass();
|
|
const operation = await operationFor(job.deploymentId);
|
|
expect(operation).toBeDefined();
|
|
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
const payload = getHostOperation(operation!.id)!.payload as { files: { contentBase64: string; sha256: string }[] };
|
|
expect(payload.files).toContainEqual(expect.objectContaining({ contentBase64: content.toString('base64'), sha256: latest.sha256 }));
|
|
});
|
|
|
|
it('prepares only one launch while the single startup slot is reserved', async () => {
|
|
const first = await queue();
|
|
const second = await queue();
|
|
const paused = await pausePreparation();
|
|
const pass = runPass();
|
|
await paused.ready;
|
|
expect(paused.spy).toHaveBeenCalledTimes(1);
|
|
const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
|
|
await processNativeProvisioning(() => null, { wait: false });
|
|
expect(paused.spy).toHaveBeenCalledTimes(1);
|
|
paused.release();
|
|
await pass;
|
|
expect(await operationFor(first.deploymentId)).toBeDefined();
|
|
expect(await operationFor(second.deploymentId)).toBeUndefined();
|
|
});
|
|
|
|
it('keeps a prepared launch queued while its host is disabled or re-enabling', async () => {
|
|
const job = await queue();
|
|
const paused = await pausePreparation();
|
|
const pass = runPass();
|
|
await paused.ready;
|
|
const { updateHost } = await import('../../src/lib/server/hosts/index.js');
|
|
updateHost(hostId, { mode: 'disabled' });
|
|
paused.release();
|
|
await pass;
|
|
expect(await operationFor(job.deploymentId)).toBeUndefined();
|
|
expect(updateHost(hostId, { mode: 'active' })).toMatchObject({ mode: 'disabled', readiness: 'preparing' });
|
|
await runPass();
|
|
expect(await operationFor(job.deploymentId)).toBeUndefined();
|
|
});
|
|
|
|
it('rechecks a reduced startup limit before submitting prepared files', async () => {
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
const first = await queue();
|
|
await runPass();
|
|
expect(await operationFor(first.deploymentId)).toBeDefined();
|
|
const second = await queue();
|
|
const paused = await pausePreparation();
|
|
const pass = runPass();
|
|
await paused.ready;
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
|
|
paused.release();
|
|
await pass;
|
|
expect(await operationFor(second.deploymentId)).toBeUndefined();
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
await runPass();
|
|
expect(await operationFor(second.deploymentId)).toBeDefined();
|
|
});
|
|
});
|