512 lines
22 KiB
TypeScript
512 lines
22 KiB
TypeScript
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({
|
|
checkRelease: vi.fn(async () => {}),
|
|
freshRelease: () => 5354,
|
|
probeVersion: vi.fn(async () => 5354)
|
|
}));
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({
|
|
kick: vi.fn(async () => {}),
|
|
kickProvisioning: vi.fn(async () => {})
|
|
}));
|
|
|
|
let fixture: Fixture;
|
|
let hostId: string;
|
|
let libraryId: string;
|
|
const image = 'example.test/plutainer:test';
|
|
const releases: (() => void)[] = [];
|
|
const passes: Promise<void>[] = [];
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.clearAllMocks();
|
|
fixture = await newFixture([]);
|
|
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
|
|
vi.stubEnv('PLUTAINER_IMAGE', image);
|
|
vi.stubEnv('RCON_PASSWORD', 'test-password');
|
|
vi.stubEnv('EC_PUBLIC_URL', 'https://public-controller.example');
|
|
|
|
const { createHost, createEnrollment, enrollHost, heartbeat } =
|
|
await import('../../src/lib/server/hosts/index.js');
|
|
hostId = createHost({ name: 'Preparation test host', portMin: 28960, portMax: 28980 }).id;
|
|
const inventory = {
|
|
cpuCount: 32,
|
|
cpuUsagePercent: 0,
|
|
memoryTotal: 64 * 1024 ** 3,
|
|
memoryAvailable: 60 * 1024 ** 3,
|
|
protocolVersion: 1,
|
|
dockerRunning: true,
|
|
directoriesWritable: true,
|
|
storageAvailable: 1_000_000_000_000,
|
|
preparedRuntimeImages: [image]
|
|
};
|
|
enrollHost(createEnrollment(hostId).token, inventory);
|
|
const { createLibraryImport, updateHostLibraryCaches } =
|
|
await import('../../src/lib/server/game-files/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
libraryId = createLibraryImport({ hostId, game: 't5' }).id;
|
|
db()
|
|
.prepare(
|
|
"UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?"
|
|
)
|
|
.run('a'.repeat(64), new Date().toISOString(), libraryId);
|
|
updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: libraryId, status: 'ready', manifestSha256: 'a'.repeat(64) }
|
|
]);
|
|
heartbeat(hostId, { ...inventory, libraries: [{ id: libraryId, state: 'ready' }] });
|
|
});
|
|
|
|
afterEach(async () => {
|
|
for (const release of releases.splice(0)) release();
|
|
await Promise.allSettled(passes.splice(0));
|
|
vi.restoreAllMocks();
|
|
await fixture.cleanup();
|
|
vi.unstubAllEnvs();
|
|
});
|
|
|
|
let queuedKeys = 0;
|
|
async function queue() {
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const result = queueNativeCreation({
|
|
game: 't5',
|
|
gameMode: 'zombie',
|
|
mapSlug: 'theater',
|
|
hostId,
|
|
libraryId,
|
|
image,
|
|
plutoKey: `saved-key-${++queuedKeys}`
|
|
});
|
|
if (!result.ok) throw new Error(result.message);
|
|
return result;
|
|
}
|
|
|
|
async function runPass() {
|
|
const { processNativeProvisioning } =
|
|
await import('../../src/lib/server/provisioning/native.js');
|
|
const pass = processNativeProvisioning(() => null);
|
|
passes.push(pass);
|
|
await pass;
|
|
}
|
|
|
|
async function operationFor(deploymentId: string) {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
return db()
|
|
.prepare(
|
|
`SELECT o.id FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id
|
|
WHERE n.deployment_id=?`
|
|
)
|
|
.get(deploymentId) as { id: string } | undefined;
|
|
}
|
|
|
|
/** Pause after real bytes are captured so mutations exercise the final admission checks. */
|
|
async function pausePreparation() {
|
|
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
|
|
const prepare = preparation.prepareNativeLaunch;
|
|
let started!: () => void;
|
|
let release!: () => void;
|
|
const ready = new Promise<void>((resolve) => {
|
|
started = resolve;
|
|
});
|
|
const gate = new Promise<void>((resolve) => {
|
|
release = resolve;
|
|
});
|
|
releases.push(release);
|
|
const spy = vi
|
|
.spyOn(preparation, 'prepareNativeLaunch')
|
|
.mockImplementationOnce(async (input) => {
|
|
const result = await prepare(input);
|
|
started();
|
|
await gate;
|
|
return result;
|
|
});
|
|
return { ready, release, spy };
|
|
}
|
|
|
|
describe('asynchronous native launch preparation', () => {
|
|
it('does not place an unsubmitted intent cancelled while host ranking is pending', async () => {
|
|
const job = await queue();
|
|
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
|
|
const evaluate = preparation.evaluateNativePlacement;
|
|
let started!: () => void;
|
|
let release!: () => void;
|
|
const ready = new Promise<void>((resolve) => {
|
|
started = resolve;
|
|
});
|
|
const gate = new Promise<void>((resolve) => {
|
|
release = resolve;
|
|
});
|
|
releases.push(release);
|
|
vi.spyOn(preparation, 'evaluateNativePlacement').mockImplementationOnce(async (input) => {
|
|
const result = await evaluate(input);
|
|
started();
|
|
await gate;
|
|
return result;
|
|
});
|
|
const pass = runPass();
|
|
await ready;
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)).toBeNull();
|
|
const { cancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String));
|
|
release();
|
|
await pass;
|
|
expect(nativeInstance(job.name)).toBeNull();
|
|
expect(await operationFor(job.deploymentId)).toBeUndefined();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
});
|
|
|
|
it('keeps controller timers and launch details responsive while preparing multiple large plugins', async () => {
|
|
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { fleetSettings, saveFleetSettings } =
|
|
await import('../../src/lib/server/settings.js');
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 });
|
|
const { script, version } = createScript({
|
|
name: 'Large launch plugin',
|
|
kind: 'plugin',
|
|
targetRelPath: 'fixture.dll',
|
|
scopeType: 'game',
|
|
scopeValue: 't5',
|
|
content: Buffer.alloc(8 * 1024 ** 2, 42)
|
|
});
|
|
db()
|
|
.prepare('UPDATE scripts SET current_version_id=? WHERE id=?')
|
|
.run(version.id, script.id);
|
|
const jobs = await Promise.all(Array.from({ length: 4 }, queue));
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
const queueCommand = hosts.queueHostOperationSerialized;
|
|
let yielded = true;
|
|
let uninterruptedCommits = 0;
|
|
const committing = vi
|
|
.spyOn(hosts, 'queueHostOperationSerialized')
|
|
.mockImplementation((...args) => {
|
|
if (!yielded) uninterruptedCommits++;
|
|
yielded = false;
|
|
setImmediate(() => {
|
|
yielded = true;
|
|
});
|
|
return queueCommand(...args);
|
|
});
|
|
const parse = JSON.parse;
|
|
const stringify = JSON.stringify;
|
|
const parsing = vi.spyOn(JSON, 'parse').mockImplementation((raw, reviver) => {
|
|
if (typeof raw === 'string' && raw.length > 1_000_000)
|
|
throw new Error('Large file payload parsed on the controller thread');
|
|
return parse(raw, reviver);
|
|
});
|
|
const encoding = vi
|
|
.spyOn(JSON, 'stringify')
|
|
.mockImplementation((...args: Parameters<typeof JSON.stringify>) => {
|
|
const encoded = stringify(...args);
|
|
if (encoded && encoded.length > 1_000_000)
|
|
throw new Error('Large file payload serialized on the controller thread');
|
|
return encoded;
|
|
});
|
|
let ticks = 0;
|
|
const timer = setInterval(() => {
|
|
ticks++;
|
|
}, 1);
|
|
try {
|
|
const pass = runPass();
|
|
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
|
for (const job of jobs)
|
|
expect(getDeployment(job.deploymentId)?.recipe?.files).toContainEqual(
|
|
expect.objectContaining({ path: 'fixture.dll', size: 8 * 1024 ** 2 })
|
|
);
|
|
await pass;
|
|
expect(ticks).toBeGreaterThan(1);
|
|
expect(committing).toHaveBeenCalledTimes(4);
|
|
expect(uninterruptedCommits).toBe(0);
|
|
for (const job of jobs) {
|
|
expect(await operationFor(job.deploymentId)).toBeDefined();
|
|
expect(getDeployment(job.deploymentId)?.recipe?.files).toContainEqual(
|
|
expect.objectContaining({ path: 'fixture.dll', sha256: version.sha256 })
|
|
);
|
|
}
|
|
} finally {
|
|
clearInterval(timer);
|
|
parsing.mockRestore();
|
|
encoding.mockRestore();
|
|
committing.mockRestore();
|
|
}
|
|
}, 30_000);
|
|
|
|
it('does not resurrect an unsubmitted creation cancelled while its files are prepared', async () => {
|
|
const job = await queue();
|
|
const paused = await pausePreparation();
|
|
const pass = runPass();
|
|
await paused.ready;
|
|
const { cancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String));
|
|
paused.release();
|
|
await pass;
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(await operationFor(job.deploymentId)).toBeUndefined();
|
|
expect(db().prepare('SELECT name FROM servers WHERE name=?').get(job.name)).toBeUndefined();
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
expect(
|
|
db().prepare('SELECT state FROM server_deployments WHERE id=?').get(job.deploymentId)
|
|
).toEqual({ state: 'cancelled' });
|
|
});
|
|
|
|
it.each(['baseline', 'config', 'script', 'plugin'] as const)(
|
|
'discards prepared bytes when the promoted %s changes before dispatch',
|
|
async (kind) => {
|
|
const { createScript, uploadNewVersion } =
|
|
await import('../../src/lib/server/fleet/scripts.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const baseline = createScript({
|
|
name: 'Launch baseline',
|
|
kind: 'baseline',
|
|
scopeType: 'game',
|
|
scopeValue: 't5',
|
|
content: Buffer.from('set rcon_password "password"\nset sv_voice "1"\n')
|
|
});
|
|
db()
|
|
.prepare('UPDATE scripts SET current_version_id=? WHERE id=?')
|
|
.run(baseline.version.id, baseline.script.id);
|
|
const isConfig = kind === 'baseline' || kind === 'config';
|
|
const { script, version } =
|
|
kind === 'baseline'
|
|
? baseline
|
|
: createScript({
|
|
releaseVersion: '1.0.0',
|
|
name: `Launch ${kind}`,
|
|
kind,
|
|
targetRelPath:
|
|
kind === 'script'
|
|
? 'fixture.gsc'
|
|
: kind === 'plugin'
|
|
? 'fixture.dll'
|
|
: undefined,
|
|
scopeType: kind === 'config' ? 'map' : 'game',
|
|
scopeValue: kind === 'config' ? 'zombie_theater' : 't5',
|
|
content: Buffer.from(isConfig ? 'set sv_voice "1"\n' : 'before promotion')
|
|
});
|
|
db()
|
|
.prepare('UPDATE scripts SET current_version_id=? WHERE id=?')
|
|
.run(version.id, script.id);
|
|
const job = await queue();
|
|
const paused = await pausePreparation();
|
|
const pass = runPass();
|
|
await paused.ready;
|
|
const content = Buffer.from(
|
|
isConfig
|
|
? 'set rcon_password "password"\nset sv_voice "0"\n'
|
|
: 'newly promoted file'
|
|
);
|
|
const latest = uploadNewVersion(script.id, content, undefined, '1.0.1');
|
|
db()
|
|
.prepare('UPDATE scripts SET current_version_id=? WHERE id=?')
|
|
.run(latest.id, script.id);
|
|
paused.release();
|
|
await pass;
|
|
expect(await operationFor(job.deploymentId)).toBeUndefined();
|
|
await runPass();
|
|
const operation = await operationFor(job.deploymentId);
|
|
expect(operation).toBeDefined();
|
|
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
const payload = getHostOperation(operation!.id)!.payload as {
|
|
files: { contentBase64: string; sha256: string }[];
|
|
};
|
|
if (isConfig) {
|
|
const { getPromotedServerConfig } =
|
|
await import('../../src/lib/server/fleet/server-config.js');
|
|
const expected = getPromotedServerConfig(
|
|
't5',
|
|
'zombie_theater',
|
|
'theater',
|
|
'test-password'
|
|
)!;
|
|
expect(payload.files).toContainEqual(
|
|
expect.objectContaining({ contentBase64: expected.toString('base64') })
|
|
);
|
|
expect(expected.toString()).toContain('set sv_voice "0"');
|
|
} else
|
|
expect(payload.files).toContainEqual(
|
|
expect.objectContaining({
|
|
contentBase64: content.toString('base64'),
|
|
sha256: latest.sha256
|
|
})
|
|
);
|
|
}
|
|
);
|
|
|
|
it('discards prepared bytes when the server key changes before dispatch', async () => {
|
|
const job = await queue();
|
|
const paused = await pausePreparation();
|
|
const pass = runPass();
|
|
await paused.ready;
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare(
|
|
`UPDATE server_keys SET pluto_key='changed-key' WHERE id IN
|
|
(SELECT key_id FROM server_key_leases WHERE server_id=? AND is_current=1)`
|
|
)
|
|
.run(job.name);
|
|
paused.release();
|
|
await pass;
|
|
expect(await operationFor(job.deploymentId)).toBeUndefined();
|
|
await runPass();
|
|
const operation = await operationFor(job.deploymentId);
|
|
expect(operation).toBeDefined();
|
|
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
expect(getHostOperation(operation!.id)!.payload).toMatchObject({
|
|
env: { PLUTO_SERVER_KEY: 'changed-key' }
|
|
});
|
|
});
|
|
|
|
it('prepares only one launch while the single startup slot is reserved', async () => {
|
|
const first = await queue();
|
|
const second = await queue();
|
|
const paused = await pausePreparation();
|
|
const pass = runPass();
|
|
await paused.ready;
|
|
expect(paused.spy).toHaveBeenCalledTimes(1);
|
|
const { processNativeProvisioning } =
|
|
await import('../../src/lib/server/provisioning/native.js');
|
|
await processNativeProvisioning(() => null, { wait: false });
|
|
expect(paused.spy).toHaveBeenCalledTimes(1);
|
|
paused.release();
|
|
await pass;
|
|
expect(await operationFor(first.deploymentId)).toBeDefined();
|
|
expect(await operationFor(second.deploymentId)).toBeUndefined();
|
|
});
|
|
|
|
it('keeps a prepared launch queued while its host is disabled or re-enabling', async () => {
|
|
const job = await queue();
|
|
const paused = await pausePreparation();
|
|
const pass = runPass();
|
|
await paused.ready;
|
|
const { updateHost } = await import('../../src/lib/server/hosts/index.js');
|
|
updateHost(hostId, { mode: 'disabled' });
|
|
paused.release();
|
|
await pass;
|
|
expect(await operationFor(job.deploymentId)).toBeUndefined();
|
|
expect(updateHost(hostId, { mode: 'active' })).toMatchObject({
|
|
mode: 'disabled',
|
|
readiness: 'preparing'
|
|
});
|
|
await runPass();
|
|
expect(await operationFor(job.deploymentId)).toBeUndefined();
|
|
});
|
|
|
|
it('rechecks a reduced startup limit before submitting prepared files', async () => {
|
|
const { fleetSettings, saveFleetSettings } =
|
|
await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
const first = await queue();
|
|
await runPass();
|
|
expect(await operationFor(first.deploymentId)).toBeDefined();
|
|
const second = await queue();
|
|
const paused = await pausePreparation();
|
|
const pass = runPass();
|
|
await paused.ready;
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
|
|
paused.release();
|
|
await pass;
|
|
expect(await operationFor(second.deploymentId)).toBeUndefined();
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
await runPass();
|
|
expect(await operationFor(second.deploymentId)).toBeDefined();
|
|
});
|
|
});
|
|
|
|
it('injects unique persistent bank credentials only into private runtime files', async () => {
|
|
const job = await queue();
|
|
await runPass();
|
|
const op = await operationFor(job.deploymentId);
|
|
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const { bankCredential, authenticateBank } = await import('../../src/lib/server/auth/bank.js');
|
|
const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const instance = nativeInstance(job.name)!;
|
|
const credential = bankCredential(instance.id);
|
|
const payload = getHostOperation(op!.id)!.payload as {
|
|
bank: { token: string; configPath: string };
|
|
files: { path: string }[];
|
|
env: Record<string, string>;
|
|
};
|
|
expect(payload.bank).toEqual({
|
|
token: credential,
|
|
configPath: 'configs/electric_cherry_bank.cfg'
|
|
});
|
|
expect(payload.files.some((file) => file.path.endsWith('electric_cherry_bank.cfg'))).toBe(
|
|
false
|
|
);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(
|
|
db()
|
|
.prepare('SELECT count(*) AS n FROM file_blobs WHERE instr(content,?) > 0')
|
|
.get(Buffer.from(credential))
|
|
).toEqual({ n: 0 });
|
|
expect(payload.env.PLUTAINER_EXTRA_ARGS).toContain(
|
|
'+exec electric_cherry_bank.cfg +set tws_server_id'
|
|
);
|
|
expect(
|
|
authenticateBank(
|
|
new Request('https://test', { headers: { Authorization: `Bearer ${credential}` } })
|
|
)
|
|
).toMatchObject({ ok: true, server: job.name, instanceId: instance.id });
|
|
expect(JSON.stringify(recipeForDeployment(job.deploymentId))).not.toContain(credential);
|
|
expect(
|
|
recipeForDeployment(job.deploymentId)!.recipe.files.some(
|
|
(file) => file.path === 'electric_cherry_bank.cfg'
|
|
)
|
|
).toBe(false);
|
|
});
|
|
|
|
it('keeps legacy agents compatible without claiming moderation configuration', async () => {
|
|
const job = await queue();
|
|
await runPass();
|
|
const op = await operationFor(job.deploymentId);
|
|
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
const payload = getHostOperation(op!.id)!.payload as Record<string, unknown>;
|
|
expect(payload.moderation).toBeUndefined();
|
|
});
|
|
|
|
it('injects generation-bound moderation credentials privately on compatible hosts', async () => {
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
hosts.heartbeat(hostId, {
|
|
...hosts.getHost(hostId)!.inventory,
|
|
capabilities: ['moderation.configure']
|
|
});
|
|
const job = await queue();
|
|
await runPass();
|
|
const op = await operationFor(job.deploymentId);
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const { moderationCredential } = await import('../../src/lib/server/moderation/credentials.js');
|
|
const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const instance = nativeInstance(job.name)!;
|
|
const credential = moderationCredential(instance.id, instance.generation);
|
|
const payload = hosts.getHostOperation(op!.id)!.payload as {
|
|
moderation: { token: string; configPath: string; url: string; generation: number };
|
|
files: { path: string }[];
|
|
env: Record<string, string>;
|
|
};
|
|
expect(payload.moderation).toEqual({
|
|
token: credential,
|
|
configPath: 'configs/electric_cherry_moderation.cfg',
|
|
url: 'https://public-controller.example/api/moderation/runtime',
|
|
generation: instance.generation
|
|
});
|
|
expect(payload.files.some((file) => file.path.endsWith('electric_cherry_moderation.cfg'))).toBe(
|
|
false
|
|
);
|
|
expect(payload.env.PLUTAINER_EXTRA_ARGS).toContain('+exec electric_cherry_moderation.cfg');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(
|
|
db()
|
|
.prepare('SELECT count(*) AS n FROM file_blobs WHERE instr(content,?) > 0')
|
|
.get(Buffer.from(credential))
|
|
).toEqual({ n: 0 });
|
|
expect(JSON.stringify(recipeForDeployment(job.deploymentId))).not.toContain(credential);
|
|
expect(JSON.stringify(hosts.listHostOperationHistory())).not.toContain(credential);
|
|
});
|