1061 lines
78 KiB
TypeScript
1061 lines
78 KiB
TypeScript
import { createHash } from 'node:crypto';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { join } from 'node:path';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
const remote = vi.hoisted(() => ({ state: 'running', players: 0, map: 'zombie_theater' }));
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
|
rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: '', status: { players: remote.players, map: remote.map, gametype: 'zom', roster: [] } })),
|
|
rconGetPlayerCount: vi.fn(async () => remote.players), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null }))
|
|
}));
|
|
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) }));
|
|
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
|
|
|
|
let fixture: Fixture; let hostId: string; let libraryId: string;
|
|
const image = 'example.test/plutainer:test';
|
|
const inventory = () => ({ cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] });
|
|
beforeEach(async () => {
|
|
vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = '';
|
|
fixture = await newFixture([]);
|
|
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
|
|
vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password');
|
|
vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', '');
|
|
const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js');
|
|
hostId = createHost({ name: 'Native Debian', portMin: 28960, portMax: 28970 }).id;
|
|
enrollHost(createEnrollment(hostId).token, inventory());
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => {
|
|
if (kind !== 'instance.inspect') throw new Error(`Unexpected operation: ${kind}`);
|
|
return { healthy: remote.state === 'running', state: remote.state, exists: true } as never;
|
|
});
|
|
});
|
|
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
|
|
|
|
async function prepareLibrary(game: 't5' | 't6' = 't5') {
|
|
const libs = await import('../../src/lib/server/game-files/index.js');
|
|
const version = libs.createLibraryImport({ hostId, game });
|
|
const bytes = Buffer.from('verified base game');
|
|
const file = { path: 'zone/base.ff', size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') };
|
|
libs.setImportManifest(hostId, version.id, [file]);
|
|
await libs.uploadImportFile(hostId, version.id, file.path, new Response(bytes).body);
|
|
const ready = await libs.publishImport(hostId, version.id); libraryId = ready.id;
|
|
libs.updateHostLibraryCaches(hostId, [{ libraryVersionId: ready.id, status: 'ready', manifestSha256: ready.manifestSha256 }]);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory());
|
|
return ready;
|
|
}
|
|
async function queue(game: 't5' | 't6' = 't5', placement: 'pinned' | 'automatic' = 'pinned') {
|
|
await prepareLibrary(game); remote.map = game === 't5' ? 'zombie_theater' : 'zm_transit';
|
|
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const result = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town',
|
|
hostId: placement === 'pinned' ? hostId : undefined, libraryId, image, plutoKey: 'saved-key' });
|
|
if (!result.ok) throw new Error(result.message);
|
|
return result;
|
|
}
|
|
async function operationFor(deploymentId: string) {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const row = db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(deploymentId) as { operation_id: string | null };
|
|
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
return row?.operation_id ? getHostOperation(row.operation_id)! : null;
|
|
}
|
|
async function advance() {
|
|
const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
|
|
await processNativeProvisioning(() => null);
|
|
}
|
|
async function acknowledge(deploymentId: string) {
|
|
const operation = await operationFor(deploymentId); if (!operation) throw new Error('Missing durable native operation');
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, { state: 'completed', result: { state: remote.state, healthy: remote.state === 'running' } });
|
|
await advance();
|
|
return operation;
|
|
}
|
|
async function detail(id: string) {
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
return getDeployment(id)!.deployment;
|
|
}
|
|
|
|
describe('native instance provisioning', () => {
|
|
it('recognizes an unplaced native intent and resumes it once after a controller restart', async () => {
|
|
const job = await queue();
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { nativeDeployment, nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
expect(job.port).toBeNull();
|
|
expect(nativeDeployment(job.deploymentId)).toBe(true);
|
|
expect(nativeInstance(job.name)).toBeNull();
|
|
expect(getDeployment(job.deploymentId)).toMatchObject({ backend: 'native', canCancelCreation: true });
|
|
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([{ deployment_id: job.deploymentId }]);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
|
|
closeDb(); vi.resetModules();
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
expect(backend.nativeDeployment(job.deploymentId)).toBe(true);
|
|
await advance();
|
|
const original = await operationFor(job.deploymentId);
|
|
expect(original).toMatchObject({ kind: 'instance.create', payload: { name: job.name, port: 28960 } });
|
|
await advance();
|
|
expect((await operationFor(job.deploymentId))?.id).toBe(original!.id);
|
|
const restartedDb = (await import('../../src/lib/server/db/index.js')).db();
|
|
expect(restartedDb.prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
|
|
expect(restartedDb.prepare('SELECT protocol FROM ec_host_ports ORDER BY protocol').all()).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]);
|
|
});
|
|
|
|
it('still rejects missing keys and stale lobby identities before creating an intent', async () => {
|
|
await prepareLibrary();
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', hostId, libraryId, image };
|
|
expect(queueNativeCreation(input)).toMatchObject({ ok: false, message: expect.stringContaining('server key') });
|
|
expect(queueNativeCreation({ ...input, plutoKey: 'saved-key', expectedName: 't5_zombie_theater_2' }))
|
|
.toMatchObject({ ok: false });
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([]);
|
|
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
});
|
|
|
|
it('accepts cancellation through the provisioning API and rejects an already completed creation', async () => {
|
|
const job = await queue();
|
|
const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js');
|
|
const response = POST({ params: { id: job.deploymentId } });
|
|
expect(response.status).toBe(200);
|
|
expect(await response.json()).toMatchObject({ ok: true, cleanupId: expect.any(String) });
|
|
const next = await queue(); await advance(); await acknowledge(next.deploymentId);
|
|
expect(POST({ params: { id: next.deploymentId } }).status).toBe(409);
|
|
expect(POST({ params: { id: job.deploymentId } }).status).toBe(200);
|
|
});
|
|
it('cancels an unsubmitted creation locally and returns the same cleanup on repeated requests', async () => {
|
|
const job = await queue();
|
|
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(getDeployment(job.deploymentId)!.canCancelCreation).toBe(true);
|
|
const cleanup = cancelCreation(job.deploymentId)!;
|
|
expect(cancelCreation(job.deploymentId)).toBe(cleanup);
|
|
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
|
expect((await detail(cleanup)).state).toBe('done');
|
|
expect(getDeployment(job.deploymentId)!.cancellation?.state).toBe('done');
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
|
await advance();
|
|
expect(await operationFor(job.deploymentId)).toBeNull();
|
|
expect(await operationFor(cleanup)).toBeNull();
|
|
});
|
|
|
|
it('cancels a submitted creation without occupancy checks, suppresses replay, and releases resources only after cleanup', async () => {
|
|
const job = await queue(); await advance(); remote.players = 4;
|
|
const original = (await operationFor(job.deploymentId))!;
|
|
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
const { heartbeat, pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] });
|
|
// A previously delivered command might still be executing when cancellation arrives.
|
|
expect(pendingHostOperations(hostId).map(op => op.id)).toContain(original.id);
|
|
const cleanup = cancelCreation(job.deploymentId)!;
|
|
const removal = (await operationFor(cleanup))!;
|
|
expect(removal.kind).toBe('instance.cancel-create');
|
|
expect(removal.payload).toEqual({ createOperationId: original.id });
|
|
expect(cancelCreation(job.deploymentId)).toBe(cleanup);
|
|
expect(pendingHostOperations(hostId).map(op => op.id)).toContain(removal.id);
|
|
expect(pendingHostOperations(hostId).map(op => op.id)).not.toContain(original.id);
|
|
await advance();
|
|
expect(nativeRequest).not.toHaveBeenCalled();
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
|
|
reportHostOperation(hostId, original.id, { state: 'completed', result: { healthy: true } });
|
|
await advance();
|
|
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
|
expect(nativeRequest).not.toHaveBeenCalled();
|
|
reportHostOperation(hostId, removal.id, { state: 'completed', result: { exists: false } });
|
|
await advance();
|
|
expect((await detail(cleanup)).state).toBe('done');
|
|
expect((await detail(job.deploymentId)).server_deleted_at).toBeTruthy();
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
|
expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined();
|
|
expect(db().prepare('SELECT retired_at FROM native_instances WHERE id=?').get(original.instanceId)).toMatchObject({ retired_at: expect.any(String) });
|
|
});
|
|
|
|
it('holds cancellation for an old or offline host without delivering an unsupported command or losing reservations', async () => {
|
|
const job = await queue(); await advance();
|
|
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
const { heartbeat, pendingHostOperations } = await import('../../src/lib/server/hosts/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const cleanup = cancelCreation(job.deploymentId)!;
|
|
expect(pendingHostOperations(hostId).filter(op => op.kind.startsWith('instance.'))).toEqual([]);
|
|
await advance();
|
|
expect((await detail(cleanup)).stage).toContain('Update the host agent');
|
|
expect((await detail(cleanup)).stage).toContain('Automatic updates are not enabled');
|
|
heartbeat(hostId, { ...inventory(), capabilities: ['agent.update'],
|
|
agentUpdate: { enabled: true, state: 'deferred', currentVersion: '1.2.2', targetVersion: '1.3.0' } });
|
|
await advance();
|
|
expect((await detail(cleanup)).stage).toContain('New command delivery is paused');
|
|
expect((await detail(cleanup)).stage).toContain('automatic updater can run');
|
|
db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00.000Z', hostId);
|
|
await advance();
|
|
expect((await detail(cleanup)).stage).toContain('unavailable');
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
|
|
heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] });
|
|
expect(pendingHostOperations(hostId).filter(op => op.kind === 'instance.cancel-create')).toHaveLength(1);
|
|
await acknowledge(cleanup);
|
|
expect((await detail(cleanup)).state).toBe('done');
|
|
});
|
|
|
|
it('replaces an occupancy-blocked removal and ignores an outstanding creation health reply', async () => {
|
|
const job = await queue(); await advance();
|
|
const operation = (await operationFor(job.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
let release!: (value: never) => void;
|
|
vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
|
const pending = advance();
|
|
try {
|
|
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const waitingRemoval = destroyServer(job.name).deploymentId!;
|
|
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
const cleanup = cancelCreation(job.deploymentId)!;
|
|
expect(cleanup).toBeTruthy();
|
|
expect((await detail(waitingRemoval)).state).toBe('cancelled');
|
|
release({ healthy: true, state: 'running' } as never);
|
|
await pending;
|
|
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
|
expect((await detail(cleanup)).state).toBe('running');
|
|
} finally { release({ healthy: true, state: 'running' } as never); await pending; }
|
|
});
|
|
|
|
it('does not offer creation cancellation after success or against a replacement instance', async () => {
|
|
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
|
const { canCancelCreation, cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(canCancelCreation(job.deploymentId)).toBe(false);
|
|
expect(cancelCreation(job.deploymentId)).toBeNull();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?").run(job.deploymentId);
|
|
db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
|
|
expect(cancelCreation(job.deploymentId)).toBeNull();
|
|
});
|
|
|
|
it('retains every native launch checkpoint and readiness check through completion and UI validation', async () => {
|
|
const job = await queue();
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { isProvisioningDetail, provisioningGroups } = await import('../../src/lib/ui/provisioning-presentation.js');
|
|
const queued = getDeployment(job.deploymentId)!;
|
|
expect(queued.progress?.phase).toBe('eligibility');
|
|
expect(queued.progress?.blocker).toBeNull();
|
|
expect(queued.backend).toBe('native');
|
|
await advance();
|
|
expect(getDeployment(job.deploymentId)!.progress?.recipeStatus).toBe('attempted');
|
|
await acknowledge(job.deploymentId);
|
|
const result = getDeployment(job.deploymentId)!;
|
|
expect(result.deployment.state).toBe('done');
|
|
expect(isProvisioningDetail(result, job.deploymentId)).toBe(true);
|
|
expect(result.progress?.checks.map(check => check.key)).toEqual(['container', 'game', 'map', 'plutonium-release', 'plutonium-version']);
|
|
expect(result.progress?.checks.every(check => check.result === 'pass')).toBe(true);
|
|
expect(result.progress?.checks[0].source).toBe('agent');
|
|
const steps = provisioningGroups(result).flatMap(group => group.steps);
|
|
expect(steps.every(step => step.state === 'done')).toBe(true);
|
|
expect(steps.some(step => step.label.includes('Komodo'))).toBe(false);
|
|
expect(result.agentOperation?.state).toBe('completed');
|
|
});
|
|
|
|
it('records agent receipts immediately and only once, even before the provisioning worker runs', async () => {
|
|
const job = await queue(); await advance();
|
|
const operation = (await operationFor(job.deploymentId))!;
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
for (const state of ['accepted', 'accepted', 'running', 'completed', 'completed']) {
|
|
reportHostOperation(hostId, operation.id, { state, result: { state: 'running', healthy: true } });
|
|
}
|
|
const result = getDeployment(job.deploymentId)!;
|
|
expect(result.deployment.state).toBe('running');
|
|
expect(result.agentOperation?.state).toBe('completed');
|
|
expect(result.events.filter(event => event.kind === 'agent')).toHaveLength(3);
|
|
expect(result.progress?.checkpoints.map(point => point.key)).toEqual(expect.arrayContaining(['agent-accepted', 'agent-running', 'agent-completed']));
|
|
expect(result.progress?.command?.state).toBe('accepted');
|
|
});
|
|
|
|
it('records an outstanding container check before awaiting the host and admits other jobs meanwhile', async () => {
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
const first = await queue(); await advance();
|
|
const operation = (await operationFor(first.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
|
|
const second = await queue();
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
let release!: (value: never) => void;
|
|
vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
|
const pending = advance();
|
|
try {
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
expect(getDeployment(first.deploymentId)!.progress?.phase).toBe('verify-container');
|
|
await vi.waitFor(async () => {
|
|
expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
|
|
}, { timeout: 5_000 });
|
|
} finally {
|
|
release({ healthy: true, state: 'running' } as never);
|
|
await pending;
|
|
}
|
|
});
|
|
|
|
it('admits a later request on the next scheduler pass while an earlier native probe is pending', async () => {
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
const first = await queue(); await advance();
|
|
const operation = (await operationFor(first.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
let release!: (value: never) => void;
|
|
vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
|
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
|
|
await processProvisioning({ wait: false });
|
|
try {
|
|
const second = await queue();
|
|
await processProvisioning({ wait: false });
|
|
await vi.waitFor(async () => {
|
|
expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
|
|
}, { timeout: 5_000 });
|
|
expect(vi.mocked(nativeRequest)).toHaveBeenCalledTimes(1);
|
|
} finally {
|
|
release({ healthy: true, state: 'running' } as never);
|
|
await processProvisioning();
|
|
}
|
|
});
|
|
|
|
it('ignores a late health result after removal retires the operation', async () => {
|
|
const job = await queue(); await advance();
|
|
const operation = (await operationFor(job.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
|
|
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
|
|
let release!: (value: never) => void;
|
|
vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
|
const pending = advance();
|
|
try {
|
|
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE server_deployments SET state='cancelled',server_deleted_at=? WHERE id=?")
|
|
.run(new Date().toISOString(), job.deploymentId);
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const before = getDeployment(job.deploymentId)!;
|
|
release({ healthy: true, state: 'running' } as never);
|
|
await pending;
|
|
const after = getDeployment(job.deploymentId)!;
|
|
expect(after.deployment.state).toBe('cancelled');
|
|
expect(after.progress).toEqual(before.progress);
|
|
expect(after.activity).toEqual(before.activity);
|
|
} finally { release({ healthy: true, state: 'running' } as never); await pending; }
|
|
});
|
|
|
|
it('records map mismatch evidence while retaining passed container and game checks', async () => {
|
|
const job = await queue(); await advance(); remote.map = 'zombie_moon';
|
|
await acknowledge(job.deploymentId);
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const result = getDeployment(job.deploymentId)!;
|
|
expect(result.deployment.state).toBe('awaiting-health');
|
|
expect(result.progress?.phase).toBe('verify-map');
|
|
expect(result.progress?.checks).toEqual(expect.arrayContaining([
|
|
expect.objectContaining({ key: 'container', result: 'pass' }),
|
|
expect.objectContaining({ key: 'game', result: 'pass' }),
|
|
expect.objectContaining({ key: 'map', result: 'waiting', actual: 'zombie_moon', expected: 'zombie_theater' })
|
|
]));
|
|
});
|
|
|
|
it('selects the latest game files automatically and rejects stale explicit selections', async () => {
|
|
const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
|
|
expect(() => nativeSelection({ hostId, image }, 't5')).toThrow('Upload T5 game files');
|
|
const old = await prepareLibrary('t5');
|
|
expect(nativeSelection({ hostId, image }, 't5').libraryId).toBe(old.id);
|
|
const latest = await prepareLibrary('t5');
|
|
expect(nativeSelection({ hostId, image }, 't5').libraryId).toBe(latest.id);
|
|
expect(() => nativeSelection({ hostId, image, libraryId: old.id }, 't5')).toThrow('Refresh the launch preview');
|
|
});
|
|
|
|
it('rejects a launch preview made before newer game files were published without reserving a server', async () => {
|
|
await prepareLibrary('t5');
|
|
const { previewNativeCreation, queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const preview = previewNativeCreation({ hostId, image, game: 't5', mapSlug: 'theater' });
|
|
await prepareLibrary('t5');
|
|
const result = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', previewId: preview.id, plutoKey: 'new-key' });
|
|
expect(result).toMatchObject({ ok: false, message: expect.stringContaining('Refresh the launch preview') });
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({ count: 0 });
|
|
expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ count: 0 });
|
|
});
|
|
|
|
it.each(['pinned', 'automatic'] as const)('keeps an unplaced %s launch pinned to its accepted files during cleanup', async placement => {
|
|
const job = await queue('t5', placement);
|
|
const acceptedId = libraryId;
|
|
const latest = await prepareLibrary('t5');
|
|
expect(latest.id).not.toBe(acceptedId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { reconcileHostLibraryCleanup, cleanupControllerLibraries, getLibraryVersion, updateHostLibraryCaches }
|
|
= await import('../../src/lib/server/game-files/index.js');
|
|
const accepted = getLibraryVersion(acceptedId)!;
|
|
heartbeat(hostId, { ...inventory(), capabilities: ['library.remove'] });
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
|
|
reconcileHostLibraryCleanup(hostId);
|
|
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=?")
|
|
.all(acceptedId)).toEqual([]);
|
|
|
|
// The unplaced intent also protects the controller's copy when every
|
|
// host reports it absent; its accepted version remains downloadable.
|
|
updateHostLibraryCaches(hostId, [{ libraryVersionId: acceptedId, status: 'missing' }]);
|
|
await cleanupControllerLibraries();
|
|
expect(db().prepare('SELECT version_id FROM game_file_cleanup WHERE version_id=?').get(acceptedId)).toBeUndefined();
|
|
const { stat } = await import('node:fs/promises');
|
|
expect((await stat(join(fixture.root, 'libraries', 't5', acceptedId, 'zone/base.ff'))).isFile()).toBe(true);
|
|
updateHostLibraryCaches(hostId, [{ libraryVersionId: acceptedId, status: 'ready', manifestSha256: accepted.manifestSha256 }]);
|
|
await advance();
|
|
expect((await operationFor(job.deploymentId))?.payload.libraryId).toBe(acceptedId);
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)?.library_id).toBe(acceptedId);
|
|
});
|
|
|
|
it.each(['t5', 't6'] as const)('queues and verifies a %s deployment without Komodo', async game => {
|
|
const job = await queue(game); await advance();
|
|
const operation = await operationFor(job.deploymentId);
|
|
expect(job.port).toBeNull();
|
|
expect(operation).toMatchObject({ kind: 'instance.create', hostId, generation: 1, state: 'pending', payload: { game, libraryId, image, port: 28960 } });
|
|
expect(operation?.payload.files).toEqual(expect.arrayContaining([expect.objectContaining({ path: 'configs/electric_cherry_map.cfg' })]));
|
|
await acknowledge(job.deploymentId);
|
|
expect((await detail(job.deploymentId)).state).toBe('done');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT protocol FROM ec_host_ports WHERE instance_id=? ORDER BY protocol').all(operation!.instanceId)).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]);
|
|
});
|
|
it('reconciles the same operation after controller restart and an ambiguous remote result', async () => {
|
|
const job = await queue(); await advance();
|
|
const original = (await operationFor(job.deploymentId))!;
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, original.id, { state: 'unknown', error: 'Reply lost after execution' });
|
|
const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb();
|
|
const { interruptDeployments } = await import('../../src/lib/server/provisioning/history.js'); interruptDeployments();
|
|
await advance(); await advance();
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
|
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.create'").get()).toEqual({ count: 1 });
|
|
expect((await detail(job.deploymentId)).state).toBe('running');
|
|
await acknowledge(job.deploymentId);
|
|
expect((await detail(job.deploymentId)).state).toBe('done');
|
|
});
|
|
it('holds disconnected instances as unknown without replacement and resumes the original operation', async () => {
|
|
const job = await queue(); await advance();
|
|
const original = (await operationFor(job.deploymentId))!;
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00Z', hostId);
|
|
await advance();
|
|
expect(db().prepare('SELECT container_state FROM servers WHERE name=?').get(job.name)).toEqual({ container_state: 'unknown' });
|
|
expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory());
|
|
await acknowledge(job.deploymentId);
|
|
expect((await detail(job.deploymentId)).state).toBe('done');
|
|
});
|
|
it('rebuilds only an empty instance and advances its generation after the acknowledged command', async () => {
|
|
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
|
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
|
const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
|
|
remote.players = 2; await advance();
|
|
expect(await operationFor(rebuild.deploymentId)).toBeNull();
|
|
remote.players = 0; await advance();
|
|
const operation = (await operationFor(rebuild.deploymentId))!;
|
|
expect(operation).toMatchObject({ kind: 'instance.rebuild', generation: 1, payload: { nextGeneration: 2, libraryId } });
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)?.generation).toBe(1);
|
|
await acknowledge(rebuild.deploymentId);
|
|
expect(nativeInstance(job.name)?.generation).toBe(2);
|
|
expect((await detail(rebuild.deploymentId)).state).toBe('done');
|
|
await advance();
|
|
expect(nativeInstance(job.name)?.generation).toBe(2);
|
|
});
|
|
it('holds a rebuild when players join while its launch files are being prepared', async () => {
|
|
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
|
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
|
const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
|
|
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
|
|
const prepare = preparation.prepareNativeLaunch;
|
|
let started!: () => void;
|
|
let release!: () => void;
|
|
const ready = new Promise<void>(resolve => { started = resolve; });
|
|
const gate = new Promise<void>(resolve => { release = resolve; });
|
|
const spy = vi.spyOn(preparation, 'prepareNativeLaunch').mockImplementationOnce(async input => {
|
|
const result = await prepare(input);
|
|
started(); await gate;
|
|
return result;
|
|
});
|
|
const pending = advance();
|
|
try {
|
|
await ready;
|
|
remote.players = 2;
|
|
release(); await pending;
|
|
expect(await operationFor(rebuild.deploymentId)).toBeNull();
|
|
remote.players = 0;
|
|
await advance();
|
|
expect((await operationFor(rebuild.deploymentId))?.kind).toBe('instance.rebuild');
|
|
} finally { release(); await pending; spy.mockRestore(); }
|
|
});
|
|
it('finishes native rebuild verification without waiting for its own recovery slot to be released', async () => {
|
|
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
|
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
|
const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
|
|
await advance();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
|
const actual = await vi.importActual<typeof import('../../src/lib/server/integrations/plutonium.js')>('../../src/lib/server/integrations/plutonium.js');
|
|
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' });
|
|
vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
|
|
// Version evidence must follow the submitted command, even on a fast test host.
|
|
db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
|
|
db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
|
|
await acknowledge(rebuild.deploymentId);
|
|
expect((await detail(job.deploymentId)).state).toBe('done');
|
|
expect((await detail(rebuild.deploymentId)).state).toBe('done');
|
|
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId))
|
|
.toEqual({ healthy_at: expect.any(String) });
|
|
expect(actual.runningVersions().find(version => version.server === job.name)?.revision).toBe(5354);
|
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
|
const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
|
|
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
|
|
expect(serverObservation(getServer(job.name)!).health).not.toBe('provisioning');
|
|
expect(fleetOperations().filter(operation => operation.server === job.name && operation.state !== 'done')).toEqual([]);
|
|
});
|
|
it('resumes a rebuild waiting on version evidence after controller restart without rebuilding again', async () => {
|
|
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
|
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
|
|
const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
|
|
await advance();
|
|
const operation = (await operationFor(rebuild.deploymentId))!;
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const actual = await vi.importActual<typeof import('../../src/lib/server/integrations/plutonium.js')>('../../src/lib/server/integrations/plutonium.js');
|
|
const komodo = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' });
|
|
const submittedAt = new Date(Date.now() - 1000).toISOString();
|
|
db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(submittedAt, rebuild.deploymentId);
|
|
db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(submittedAt, rebuild.deploymentId);
|
|
vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({ revision: null, error: 'Version response unavailable' });
|
|
vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
|
|
await acknowledge(rebuild.deploymentId);
|
|
expect(await detail(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: 'Waiting to verify the Plutonium runtime revision.' });
|
|
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId)).toEqual({ healthy_at: null });
|
|
closeDb();
|
|
vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
|
|
await advance();
|
|
expect((await detail(rebuild.deploymentId)).state).toBe('done');
|
|
expect((await operationFor(rebuild.deploymentId))!.id).toBe(operation.id);
|
|
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
|
|
});
|
|
it('blocks new admission on draining hosts and retires existing instances only when empty', async () => {
|
|
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
|
const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(hostId, { mode: 'draining' });
|
|
const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
|
|
expect(() => nativeSelection({ hostId, libraryId, image }, 't5')).toThrow(/draining/);
|
|
remote.players = 2; await advance();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const retirement = db().prepare("SELECT id FROM server_deployments WHERE action='destroy' AND server=?").get(job.name) as { id: string };
|
|
expect(retirement).toBeTruthy(); expect(await operationFor(retirement.id)).toBeNull();
|
|
remote.players = 0; await advance();
|
|
const remove = await acknowledge(retirement.id);
|
|
expect(remove.kind).toBe('instance.remove');
|
|
expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined();
|
|
expect(db().prepare('SELECT * FROM ec_host_ports WHERE instance_id=?').all(remove.instanceId)).toEqual([]);
|
|
const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js');
|
|
expect(getLibraryVersion(libraryId)?.state).toBe('ready');
|
|
});
|
|
it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => {
|
|
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const before = Buffer.from('original bytes on the host'); const after = Buffer.from('new campaign bytes');
|
|
db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote',?)").run(new Date().toISOString());
|
|
db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign',?,1,'pending')").run(job.name);
|
|
const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after) VALUES('campaign',?,'updated','hud.gsc',?)").run(job.name, after).lastInsertRowid);
|
|
const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
|
|
queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign', server: job.name, kind: 'script', relPath: 'hud.gsc', content: after, inventorySource: 'test', deploymentFileId: fileId });
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const restart = requestRestart(job.name, { kind: 'campaign', id: 'campaign' });
|
|
remote.players = 2; await advance();
|
|
const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string };
|
|
expect(staged.stage_operation_id).toBeTruthy();
|
|
const { getHostOperation, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
const stage = getHostOperation(staged.stage_operation_id)!;
|
|
expect(stage.kind).toBe('files.stage'); expect(await operationFor(restart.id)).toBeNull();
|
|
expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('pending');
|
|
reportHostOperation(hostId, stage.id, { state: 'completed', result: { state: 'staged' } });
|
|
await advance(); expect(await operationFor(restart.id)).toBeNull();
|
|
remote.players = 0; await advance();
|
|
const activation = (await operationFor(restart.id))!;
|
|
expect(activation).toMatchObject({ kind: 'instance.restart', generation: 1, payload: { stageId: restart.id } });
|
|
expect(db().prepare('SELECT content_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null });
|
|
const files = stage.payload.files as Array<{ id: number; path: string }>;
|
|
reportHostOperation(hostId, activation.id, { state: 'completed', result: { before: [{ id: files[0].id, path: files[0].path, contentBase64: before.toString('base64') }] } });
|
|
await advance();
|
|
expect((await detail(restart.id)).state).toBe('done');
|
|
expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('applied');
|
|
expect(db().prepare('SELECT content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex') });
|
|
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe(createHash('sha256').update(after).digest('hex'));
|
|
});
|
|
it('atomically queues mixed-game collections and reserves per-host ports when dispatched', async () => {
|
|
const t5 = await prepareLibrary('t5'); const t6 = await prepareLibrary('t6');
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const blueprint = saveBlueprint({ name: 'Both games', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }, { game: 't6', mapSlug: 'town', count: 1 }] });
|
|
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
|
const preview = previewCollection(blueprint.id, 'fill', { hostId, libraryIds: { t5: t5.id, t6: t6.id }, image });
|
|
expect(preview.ready).toBe(true);
|
|
expect(preview.servers.map(server => server.port)).toEqual([null, null, null]);
|
|
const queued = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) });
|
|
expect(queueCollection({ previewId: preview.id }).collectionId).toBe(queued.collectionId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 });
|
|
await advance();
|
|
const instances = db().prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port').all();
|
|
expect(instances).toEqual([{ host_id: hostId, library_id: t5.id, port: 28960 }, { host_id: hostId, library_id: t5.id, port: 28961 }, { host_id: hostId, library_id: t6.id, port: 28962 }]);
|
|
const { createHost, createEnrollment, enrollHost, hostPortAvailable } = await import('../../src/lib/server/hosts/index.js');
|
|
const secondHost = createHost({ name: 'Second Debian' }); enrollHost(createEnrollment(secondHost.id).token, inventory());
|
|
const { updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
|
|
updateHostLibraryCaches(secondHost.id, [{ libraryVersionId: t5.id, status: 'ready', manifestSha256: t5.manifestSha256 }]);
|
|
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'moon', hostId: secondHost.id, libraryId: t5.id, image, plutoKey: 'second-host-key' });
|
|
expect(second.ok).toBe(true); if (!second.ok) throw new Error(second.message);
|
|
expect(second.port).toBeNull();
|
|
await advance();
|
|
expect((await operationFor(second.deploymentId))?.payload.port).toBe(28960);
|
|
expect(hostPortAvailable(hostId, 28960)).toBe(false);
|
|
expect(hostPortAvailable(secondHost.id, 28960)).toBe(false);
|
|
expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ count: 8 });
|
|
});
|
|
it('reads remote instance files through the agent including missing files and epoch timestamps', async () => {
|
|
const job = await queue(); await advance();
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
const remoteRead = vi.mocked(backend.nativeRequest);
|
|
remoteRead.mockImplementation(async (_instance, kind, payload) => {
|
|
if (payload?.path === 'missing') return (kind === 'files.read' ? { contentBase64: null } : null) as never;
|
|
if (kind === 'files.stat') return { size: 15, mtime: 1_790_000_000, isFile: true, isDirectory: false } as never;
|
|
if (kind === 'files.read') return { contentBase64: Buffer.from('agent file bytes').toString('base64') } as never;
|
|
if (kind === 'files.list') return { entries: [{ name: 'configs', isDirectory: true, isFile: false }] } as never;
|
|
throw new Error('Unexpected file operation');
|
|
});
|
|
const files = await import('../../src/lib/server/backends/files.js');
|
|
const root = join(fixture.t5, job.name);
|
|
expect((await files.readFile(join(root, 'configs/server.cfg'))).toString()).toBe('agent file bytes');
|
|
const info = await files.stat(join(root, 'configs/server.cfg'));
|
|
expect(info.mtime.getTime()).toBe(1_790_000_000_000); expect(info.isFile()).toBe(true);
|
|
expect(await files.exists(join(root, 'missing'))).toBe(false);
|
|
await expect(files.readFile(join(root, 'missing'))).rejects.toMatchObject({ code: 'ENOENT' });
|
|
const entries = await files.readdir(root, { withFileTypes: true });
|
|
expect(entries[0].name).toBe('configs'); expect(entries[0].isDirectory()).toBe(true);
|
|
expect(remoteRead).toHaveBeenCalledWith(expect.objectContaining({ server: job.name }), 'files.read', { path: 'configs/server.cfg' });
|
|
const { stat } = await import('node:fs/promises');
|
|
await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' });
|
|
});
|
|
it('retires an agent-confirmed missing container without waiting on nonexistent RCON', async () => {
|
|
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
|
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const removal = destroyServer(job.name); if (!removal.ok || !removal.deploymentId) throw new Error('Removal was not queued');
|
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconStatusResult).mockClear();
|
|
remote.state = 'missing'; await advance();
|
|
expect((await operationFor(removal.deploymentId))?.kind).toBe('instance.remove');
|
|
expect(rconStatusResult).not.toHaveBeenCalled();
|
|
await acknowledge(removal.deploymentId);
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)).toBeNull();
|
|
expect((await detail(removal.deploymentId)).state).toBe('done');
|
|
});
|
|
it('releases a failed activation slot without recording fictitious file changes', async () => {
|
|
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('preflight-failed','promote',?)").run(new Date().toISOString());
|
|
db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('preflight-failed',?,1,'pending')").run(job.name);
|
|
const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path) VALUES('preflight-failed',?,'updated','hud.gsc')").run(job.name).lastInsertRowid);
|
|
const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
|
|
queueFileChange({ sourceKind: 'campaign', sourceId: 'preflight-failed', server: job.name, kind: 'script', relPath: 'hud.gsc', content: Buffer.from('new bytes'), inventorySource: 'test', deploymentFileId: fileId });
|
|
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
|
const restart = requestRestart(job.name, { kind: 'campaign', id: 'preflight-failed' });
|
|
await advance();
|
|
const stage = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string };
|
|
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
|
|
reportHostOperation(hostId, stage.stage_operation_id, { state: 'completed', result: { state: 'staged' } });
|
|
await advance();
|
|
const activation = (await operationFor(restart.id))!;
|
|
expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('applying');
|
|
reportHostOperation(hostId, activation.id, { state: 'failed', error: 'Target changed after campaign review; no activation occurred' });
|
|
await advance();
|
|
expect((await detail(restart.id)).state).toBe('failed');
|
|
expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('pending');
|
|
expect(db().prepare('SELECT COUNT(*) AS count FROM restart_rollout WHERE healthy_at IS NULL').get()).toEqual({ count: 0 });
|
|
expect(db().prepare('SELECT content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, sha256_before: null });
|
|
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
|
|
expect(getServerScript(job.name, 'hud.gsc')).toBeUndefined();
|
|
});
|
|
it('cancels an unplaced native launch locally without reserving ports or submitting a create', async () => {
|
|
const job = await queue();
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
expect(nativeInstance(job.name)).toBeNull();
|
|
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const cancelled = destroyServer(job.name); expect(cancelled.ok).toBe(true);
|
|
await advance();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect((await detail(job.deploymentId)).state).toBe('cancelled');
|
|
expect(nativeInstance(job.name)).toBeNull();
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
|
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind LIKE 'instance.%'").get()).toEqual({ count: 0 });
|
|
const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js');
|
|
expect(getLibraryVersion(libraryId)?.state).toBe('ready');
|
|
const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
|
expect(getServerKey(job.name)).toBe('saved-key');
|
|
});
|
|
});
|
|
|
|
describe('automatic capacity admission and placement', () => {
|
|
const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'key' };
|
|
async function secondHost(overrides = {}) {
|
|
const api = await import('../../src/lib/server/hosts/index.js');
|
|
const libs = await import('../../src/lib/server/game-files/index.js');
|
|
const host = api.createHost({ name: 'Second host', publicAddress: 'second.example', portMin: 28960, portMax: 28970 });
|
|
api.updateHost(host.id, { publicAddress: 'second.example' });
|
|
api.enrollHost(api.createEnrollment(host.id).token, { ...inventory(), ...overrides });
|
|
const version = libs.getLibraryVersion(libraryId)!;
|
|
libs.updateHostLibraryCaches(host.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: version.manifestSha256 }]);
|
|
return host;
|
|
}
|
|
it('admits without a host, then selects the least loaded eligible host and its network defaults', async () => {
|
|
await prepareLibrary();
|
|
const other = await secondHost({ cpuUsagePercent: 0 });
|
|
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const result = queueCreateServer(input);
|
|
expect(result.ok).toBe(true); if (!result.ok) return;
|
|
expect(nativeInstance(result.name)).toBeNull();
|
|
expect(result.port).toBeNull();
|
|
await advance();
|
|
expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id);
|
|
expect(nativeInstance(result.name)).toMatchObject({ host_id: other.id, public_address: 'second.example' });
|
|
expect(recipeForDeployment(result.deploymentId)?.recipe.native?.hostId).toBe(other.id);
|
|
// Once submitted, even a now-offline host cannot trigger another placement.
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(other.id);
|
|
await advance();
|
|
expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id);
|
|
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
|
|
});
|
|
it('moves an unsubmitted reservation off an offline host and preserves explicit network overrides', async () => {
|
|
await prepareLibrary(); await secondHost();
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const result = queueNativeCreation({ ...input, publicAddress: 'nat.example', publicPort: 30000 });
|
|
expect(result.ok).toBe(true); if (!result.ok) return;
|
|
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId);
|
|
await advance();
|
|
expect(nativeInstance(result.name)).toMatchObject({ public_address: 'nat.example', public_port: 30000 });
|
|
expect((await operationFor(result.deploymentId))?.hostId).not.toBe(hostId);
|
|
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toEqual([]);
|
|
});
|
|
it.each([
|
|
{ cpuUsagePercent: undefined }, { memoryAvailable: undefined }, { cpuCount: 0 },
|
|
{ cpuUsagePercent: 99 }, { memoryAvailable: 1024 ** 3 }
|
|
])('accepts an unplaced job and waits for unavailable resources: %j', async overrides => {
|
|
await prepareLibrary();
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), ...overrides });
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const queued = queueNativeCreation(input);
|
|
expect(queued).toMatchObject({ ok: true, port: null });
|
|
if (!queued.ok) throw new Error(queued.message);
|
|
await advance();
|
|
expect(await operationFor(queued.deploymentId)).toBeNull();
|
|
expect((await detail(queued.deploymentId)).stage).toContain('capacity');
|
|
expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
|
});
|
|
it('rechecks preview admission without binding the request to the preview host or port', async () => {
|
|
await prepareLibrary();
|
|
const other = await secondHost();
|
|
const { updateHost } = await import('../../src/lib/server/hosts/index.js');
|
|
updateHost(other.id, { portMin: 30000, portMax: 30002 });
|
|
const { previewNativeCreation, queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const preview = previewNativeCreation(input);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId);
|
|
const result = queueNativeCreation({ ...input, previewId: preview.id });
|
|
expect(result).toMatchObject({ ok: true, name: preview.name, port: null });
|
|
expect(queueNativeCreation({ ...input, previewId: preview.id })).toEqual(result);
|
|
if (!result.ok) throw new Error(result.message);
|
|
await advance();
|
|
expect((await operationFor(result.deploymentId))?.payload.port).toBe(30000);
|
|
});
|
|
it('queues the whole collection and waits when capacity shrinks after its preview', async () => {
|
|
await prepareLibrary();
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
|
const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
|
|
const preview = previewCollection(blueprint.id);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 });
|
|
const result = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) });
|
|
expect(result).toMatchObject({ ok: true, collectionId: expect.any(String) });
|
|
await advance();
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
|
|
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
|
|
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toHaveLength(1);
|
|
expect(db().prepare('SELECT * FROM native_creation_intents').all()).toHaveLength(2);
|
|
});
|
|
it.each([
|
|
{ resource: 'CPU', measurements: { cpuCount: 4, cpuUsagePercent: 82.5 }, over: { cpuUsagePercent: 82.51 } },
|
|
{ resource: 'memory', measurements: { memoryAvailable: 1024 ** 3 + 500_000_000 }, over: { memoryAvailable: 1024 ** 3 + 499_999_999 } }
|
|
])('reserves only the requested host $resource headroom', async ({ measurements, over }) => {
|
|
await prepareLibrary();
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js');
|
|
const options = { hostId, libraryId, image };
|
|
heartbeat(hostId, { ...inventory(), ...measurements });
|
|
// Includes the new server's 0.5-core / 1-GiB bootstrap reservation.
|
|
expect(choosePlacement(options).host.id).toBe(hostId);
|
|
expect(choosePlacement({ ...options, thorough: true }).host.id).toBe(hostId);
|
|
heartbeat(hostId, { ...inventory(), ...measurements, ...over });
|
|
expect(() => choosePlacement(options)).toThrow('capacity');
|
|
expect(() => choosePlacement({ ...options, thorough: true })).toThrow('capacity');
|
|
});
|
|
it('dispatches a waiting instance after calibration lowers the cold-start allowance', async () => {
|
|
const first = await queue();
|
|
await advance(); await acknowledge(first.deploymentId);
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const { recordInstanceResources, capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const measured = { ...inventory(), cpuCount: 1, cpuUsagePercent: 50 };
|
|
heartbeat(hostId, measured);
|
|
const next = queueNativeCreation(input);
|
|
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
|
|
await advance();
|
|
expect(await operationFor(next.deploymentId)).toBeNull();
|
|
const instance = nativeInstance(first.name)!;
|
|
for (let minute = 0; minute < 25; minute++) {
|
|
recordInstanceResources(hostId, { ...measured, instanceResources: [{ instanceId: instance.id, generation: 1,
|
|
uptimeSeconds: minute < 4 ? minute * 60 : 600, cpuCores: 0.1, memoryBytes: 256 * 1024 ** 2 }] },
|
|
new Date(Date.now() - (25 - minute) * 60_000).toISOString());
|
|
}
|
|
expect(capacityCalibration(hostId).startup.source).toBe('host');
|
|
await advance();
|
|
expect(await operationFor(next.deploymentId)).not.toBeNull();
|
|
});
|
|
it('queues a collection at running demand and releases startup reservations as live usage arrives', async () => {
|
|
await prepareLibrary();
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
|
const GiB = 1024 ** 3;
|
|
const measured = { ...inventory(), cpuCount: 4, cpuUsagePercent: 5, memoryTotal: 4 * GiB, memoryAvailable: 3.75 * GiB };
|
|
heartbeat(hostId, measured);
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
|
|
const insert = db().prepare(`INSERT INTO ec_resource_allowances
|
|
(scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,30,?,?,?)`);
|
|
const now = new Date().toISOString();
|
|
const start = new Date(Date.now() - 30 * 60_000).toISOString();
|
|
insert.run(hostId, 'startup', 2, 2 * GiB, start, now, now);
|
|
insert.run(hostId, 'running', 0.2, GiB / 4, start, now, now);
|
|
const blueprint = saveBlueprint({ name: 'Staggered lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 3 }] });
|
|
const preview = previewCollection(blueprint.id);
|
|
expect(preview.ready).toBe(true);
|
|
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) }).ok).toBe(true);
|
|
await advance();
|
|
const creates = () => db().prepare("SELECT instance_id FROM ec_host_operations WHERE kind='instance.create'").all() as { instance_id: string }[];
|
|
expect(creates()).toHaveLength(1);
|
|
await advance();
|
|
expect(creates()).toHaveLength(1); // No heartbeat yet: the first launch still reserves startup resources.
|
|
heartbeat(hostId, { ...measured, cpuUsagePercent: 10, memoryAvailable: 3.5 * GiB,
|
|
instanceResources: [{ instanceId: creates()[0].instance_id, generation: 1, uptimeSeconds: 60, cpuCores: 0.2, memoryBytes: GiB / 4 }] });
|
|
await advance();
|
|
expect(creates()).toHaveLength(2); // Its actual usage replaces the peak even before health verification.
|
|
});
|
|
it('reserves queued lobby identities without host resources and releases them on cancellation', async () => {
|
|
await prepareLibrary();
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
const first = queueNativeCreation(input);
|
|
expect(first.ok).toBe(true); if (!first.ok) return;
|
|
const second = queueNativeCreation(input);
|
|
expect(second).toMatchObject({ ok: true, name: 't5_zombie_theater_2', port: null });
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
|
|
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
|
|
cancelCreation(first.deploymentId);
|
|
expect(queueNativeCreation(input)).toMatchObject({ ok: true, name: first.name, port: null });
|
|
});
|
|
it.each(['memory', 'cpu'] as const)('dispatches immediately after %s recovers instead of waiting for five-minute history', async resource => {
|
|
await prepareLibrary();
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const GiB = 1024 ** 3;
|
|
// A recently busy 4 GiB host is now mostly empty, as in the reported
|
|
// ec-2 inventory. Admission must not pass only to stall in dispatch.
|
|
heartbeat(hostId, { ...inventory(), cpuCount: 4, memoryTotal: 4 * GiB, memoryAvailable: Math.floor(3.8 * GiB) });
|
|
for (let minute = 1; minute <= 4; minute++) db().prepare(`INSERT INTO ec_host_resource_samples
|
|
(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,4,?,?,?)`)
|
|
.run(hostId, new Date(Date.now() - minute * 60_000).toISOString(), resource === 'cpu' ? 100 : 10,
|
|
4 * GiB, resource === 'memory' ? GiB / 8 : Math.floor(3.8 * GiB));
|
|
const jobs = Array.from({ length: 3 }, () => queueNativeCreation(input));
|
|
expect(jobs.every(job => job.ok)).toBe(true);
|
|
await advance();
|
|
expect(await operationFor(jobs[0].ok ? jobs[0].deploymentId : 'missing')).not.toBeNull();
|
|
});
|
|
it('dispatches one queued launch when learned startup costs outgrow the original batch reservation', async () => {
|
|
await prepareLibrary();
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const GiB = 1024 ** 3;
|
|
heartbeat(hostId, { ...inventory(), cpuCount: 4, memoryTotal: 4 * GiB, memoryAvailable: Math.floor(3.8 * GiB) });
|
|
const jobs = Array.from({ length: 3 }, () => queueNativeCreation(input));
|
|
expect(jobs.every(job => job.ok)).toBe(true);
|
|
// Learning increases the startup allowance after admission. All three
|
|
// no longer fit concurrently, but this otherwise empty host fits one.
|
|
db().prepare(`INSERT INTO ec_resource_allowances
|
|
(scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at)
|
|
VALUES(?,'startup',0.5,?,3,?,?,?)`).run(hostId, 2 * GiB,
|
|
new Date(Date.now() - 180_000).toISOString(), new Date().toISOString(), new Date().toISOString());
|
|
await advance();
|
|
const operations = await Promise.all(jobs.map(job => operationFor(job.ok ? job.deploymentId : 'missing')));
|
|
expect(operations.filter(Boolean)).toHaveLength(1);
|
|
expect(operations[0]?.kind).toBe('instance.create');
|
|
// The dispatched operation now consumes capacity and blocks the others,
|
|
// even though the next host heartbeat has not observed it yet.
|
|
await advance();
|
|
expect(db().prepare("SELECT * FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
|
|
const later = queueNativeCreation(input);
|
|
expect(later.ok).toBe(true); if (!later.ok) throw new Error(later.message);
|
|
await advance();
|
|
expect(await operationFor(later.deploymentId)).toBeNull();
|
|
});
|
|
it('does not charge a starting container twice when its usage is already in the heartbeat', async () => {
|
|
await prepareLibrary();
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const GiB = 1024 ** 3;
|
|
const host = { ...inventory(), cpuCount: 4, memoryTotal: 3 * GiB, memoryAvailable: 2.8 * GiB };
|
|
heartbeat(hostId, { ...host, memoryAvailable: Math.floor(host.memoryAvailable) });
|
|
const first = queueNativeCreation(input);
|
|
expect(first.ok).toBe(true); if (!first.ok) return;
|
|
await advance();
|
|
const instance = nativeInstance(first.name)!;
|
|
// 0.2 GiB OS + 0.8 GiB container are already in host usage.
|
|
// Adding a new 1 GiB server fits below 2.53 GiB usable memory.
|
|
heartbeat(hostId, { ...host, memoryAvailable: 2 * GiB, instanceResources: [{ instanceId: instance.id,
|
|
generation: instance.generation, uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
|
|
const second = queueNativeCreation(input);
|
|
expect(second).toMatchObject({ ok: true });
|
|
await advance();
|
|
expect(await operationFor(second.ok ? second.deploymentId : 'missing')).not.toBeNull();
|
|
});
|
|
it.each(['missing', 'old-generation'] as const)('keeps full reservations for a submitted launch with %s measurements', async evidence => {
|
|
await prepareLibrary();
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const GiB = 1024 ** 3;
|
|
heartbeat(hostId, { ...inventory(), memoryTotal: 3 * GiB, memoryAvailable: Math.floor(2.8 * GiB) });
|
|
const first = queueNativeCreation(input);
|
|
expect(first.ok).toBe(true); if (!first.ok) return;
|
|
await advance();
|
|
const instance = nativeInstance(first.name)!;
|
|
heartbeat(hostId, { ...inventory(), memoryTotal: 3 * GiB, memoryAvailable: 2 * GiB,
|
|
instanceResources: evidence === 'missing' ? [] : [{ instanceId: instance.id, generation: instance.generation + 1,
|
|
uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
|
|
const next = queueNativeCreation(input);
|
|
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
|
|
await advance();
|
|
expect(await operationFor(next.deploymentId)).toBeNull();
|
|
expect((await detail(next.deploymentId)).stage).toContain('3.00 GiB projected, 2.53 GiB usable');
|
|
});
|
|
it('waits when capacity disappears after admission without sending a create command', async () => {
|
|
await prepareLibrary();
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const first = queueNativeCreation(input);
|
|
expect(first.ok).toBe(true); if (!first.ok) return;
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 });
|
|
await advance();
|
|
expect(await operationFor(first.deploymentId)).toBeNull();
|
|
expect((await detail(first.deploymentId)).stage).toContain('capacity');
|
|
});
|
|
it('uses recent history at placement while keeping admission to current inventory', async () => {
|
|
await prepareLibrary();
|
|
const other = await secondHost({ cpuUsagePercent: 30 });
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare(`INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available)
|
|
VALUES(?,?,16,95,?,?)`).run(hostId, new Date(Date.now() - 60_000).toISOString(), 64 * 1024 ** 3, 60 * 1024 ** 3);
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const first = queueNativeCreation(input);
|
|
expect(first.ok).toBe(true); if (!first.ok) return;
|
|
await advance();
|
|
expect((await operationFor(first.deploymentId))?.hostId).toBe(other.id);
|
|
});
|
|
it('persists nullable CPU/memory samples and prunes old history per host', async () => {
|
|
await prepareLibrary();
|
|
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
db().prepare("INSERT INTO ec_host_resource_samples(host_id,sampled_at) VALUES(?,'2000-01-01')").run(hostId);
|
|
heartbeat(hostId, { ...inventory(), cpuUsagePercent: undefined });
|
|
closeDb();
|
|
const rows = db().prepare('SELECT * FROM ec_host_resource_samples WHERE host_id=? ORDER BY sampled_at').all(hostId) as { cpu_usage_percent: number | null; memory_available: number; sampled_at: string }[];
|
|
expect(rows.at(-1)).toMatchObject({ cpu_usage_percent: null, memory_available: 60 * 1024 ** 3 });
|
|
expect(rows.some(row => row.sampled_at === '2000-01-01')).toBe(false);
|
|
});
|
|
it('checks both port protocols and skips draining hosts', async () => {
|
|
await prepareLibrary();
|
|
const other = await secondHost();
|
|
const { reserveHostPort, updateHost } = await import('../../src/lib/server/hosts/index.js');
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
for (let port = 28960; port <= 28970; port++) reserveHostPort(hostId, 'external-' + port, { port, protocol: 'tcp' });
|
|
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
|
|
const result = queueNativeCreation(input);
|
|
expect(result.ok).toBe(true); if (!result.ok) return;
|
|
expect(nativeInstance(result.name)).toBeNull();
|
|
await advance();
|
|
expect(nativeInstance(result.name)?.host_id).toBe(other.id);
|
|
updateHost(other.id, { mode: 'draining' });
|
|
const next = queueNativeCreation(input);
|
|
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
|
|
await advance();
|
|
expect(await operationFor(next.deploymentId)).toBeNull();
|
|
});
|
|
it('spreads collection dispatch across hosts, allowing the same port on different hosts', async () => {
|
|
await prepareLibrary(); await secondHost({ cpuCount: 1, cpuUsagePercent: 0 });
|
|
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
|
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
|
|
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
|
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
|
const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
|
|
const preview = previewCollection(blueprint.id);
|
|
expect(preview.ready).toBe(true);
|
|
expect(preview.servers.map(server => server.port)).toEqual([null, null]);
|
|
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) }).ok).toBe(true);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
|
|
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
|
|
await advance();
|
|
// Concurrent ranking can select the same host before the first
|
|
// reservation commits. The held job retries against the new usage.
|
|
await advance();
|
|
expect(db().prepare('SELECT DISTINCT host_id FROM native_instances').all()).toHaveLength(2);
|
|
expect(db().prepare('SELECT port FROM native_instances').all()).toEqual([{ port: 28960 }, { port: 28960 }]);
|
|
});
|
|
});
|