josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Queue native collection launches before async placement

Browse Source
This commit is contained in:
josh
2026-10-05 11:35:50 -04:00
parent 1405114a14
commit 7c76851252
34 changed files with 1324 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/collection-launch-performance.test.ts
+29 -6
View File
@@ -79,23 +79,46 @@ describe('large native collection admission', () => {
expect(preview.ready).toBe(true);
expect(preview.servers).toHaveLength(serverCount);
expect(new Set(preview.servers.map(server => server.name)).size).toBe(serverCount);
expect(new Set(preview.servers.map(server => server.port)).size).toBe(serverCount);
expect(preview.servers.every(server => server.port === null)).toBe(true);
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(0);
const native = await import('../../src/lib/server/provisioning/native.js');
const validateSelection = vi.spyOn(native, 'nativeLaunchSelection');
manifestParses = 0;
const result = queueCollection({ previewId: preview.id,
keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) });
const queueParses = manifestParses;
expect(result).toMatchObject({ ok: true, collectionId: expect.any(String) });
expect(queueCollection({ previewId: preview.id })).toEqual(result);
expect(validateSelection).toHaveBeenCalledTimes(1);
expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: serverCount, queued: serverCount });
const instances = db().prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port').all();
expect(instances).toEqual(Array.from({ length: serverCount }, (_, index) => ({
host_id: hostId, library_id: libraryId, port: 28960 + index
})));
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(serverCount * 2);
expect(db().prepare('SELECT host_id,library_id,port FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
const intents = db().prepare('SELECT selection FROM native_creation_intents').all() as { selection: string }[];
expect(intents).toHaveLength(serverCount);
expect(intents.every(intent => JSON.parse(intent.selection).libraryId === libraryId)).toBe(true);
expect(db().prepare('SELECT * FROM server_collection_members').all()).toHaveLength(serverCount);
expect(previewParses, 'Preview must use library metadata without reading the full manifest').toBe(0);
expect(queueParses, 'Queue admission must use library metadata without reading the full manifest').toBe(0);
}, 30_000);
it.each(['newer game files', 'deleted pinned host'])('rejects %s after review before queueing any collection work', async change => {
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const { db } = await import('../../src/lib/server/db/index.js');
const blueprint = saveBlueprint({ name: 'Reviewed native selection', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
const preview = previewCollection(blueprint.id, 'fill', { hostId, libraryId, image });
if (change === 'newer game files') {
const { createLibraryImport } = await import('../../src/lib/server/game-files/index.js');
const newer = createLibraryImport({ hostId, game: 't5' });
db().prepare("UPDATE game_file_versions SET state='ready',published_at=? WHERE id=?")
.run(new Date(Date.now() + 1000).toISOString(), newer.id);
} else db().prepare('UPDATE ec_hosts SET deleted_at=? WHERE id=?').run(new Date().toISOString(), hostId);
expect(() => queueCollection({ previewId: preview.id,
keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }))
.toThrow(change === 'newer game files' ? /game files changed/ : /host was not found/);
expect(db().prepare('SELECT id FROM server_launch_collections').all()).toEqual([]);
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([]);
expect(db().prepare('SELECT server_id FROM server_keys').all()).toEqual([]);
});
});
tests/unit/collection-match-performance.test.ts
+105 -8
View File
@@ -12,10 +12,14 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
let fixture: Fixture;
let hostId: string;
let agentCredential: string;
let heartbeatInventory: Record<string, unknown>;
let releasePlacement: (() => void) | undefined;
const image = 'example.test/plutainer:test';
beforeEach(async () => {
vi.resetModules(); vi.clearAllMocks();
releasePlacement = undefined;
fixture = await newFixture([]);
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
vi.stubEnv('PLUTAINER_IMAGE', image);
@@ -31,7 +35,7 @@ beforeEach(async () => {
protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
storageAvailable: 1_000_000_000_000, runtimeImages: [image]
};
enrollHost(createEnrollment(hostId).token, inventory);
agentCredential = enrollHost(createEnrollment(hostId).token, inventory).credential;
const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
const { db } = await import('../../src/lib/server/db/index.js');
const libraries = ['t5', 't6'].map(game => {
@@ -41,7 +45,8 @@ beforeEach(async () => {
return { id, state: 'ready' as const };
});
updateHostLibraryCaches(hostId, libraries.map(library => ({ libraryVersionId: library.id, status: 'ready', manifestSha256: 'a'.repeat(64) })));
heartbeat(hostId, { ...inventory, libraries });
heartbeatInventory = { ...inventory, libraries };
heartbeat(hostId, heartbeatInventory);
const { T6_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
const now = new Date().toISOString();
db().transaction(() => {
@@ -68,6 +73,7 @@ beforeEach(async () => {
});
afterEach(async () => {
releasePlacement?.();
await new Promise<void>(resolve => setImmediate(resolve));
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
@@ -78,7 +84,44 @@ afterEach(async () => {
});
describe('matching a native collection', () => {
it('admits nine removals and ten creations without copying plugin bytes into every queued launch', async () => {
it('places queued creations and completes all exact removals during later worker passes', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 10, destroyConcurrency: 9 });
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
const { T5_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
const blueprint = saveBlueprint({ name: 'Replace nine with ten', entries: T5_ZOMBIE_MAPS.filter(map => map.slug !== 'zombietron')
.map(map => ({ game: 't5', mapSlug: map.slug, count: 1 })) });
const preview = previewCollection(blueprint.id, 'exact');
const queued = queueCollection({ previewId: preview.id,
keys: Object.fromEntries(preview.servers.map(server => [server.name!, `key-${server.name}`])) });
expect(queueCollection({ previewId: preview.id })).toEqual(queued);
expect(db().prepare('SELECT id FROM native_instances').all()).toHaveLength(9);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(18);
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
await processProvisioning();
const operations = db().prepare("SELECT id,kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.remove')")
.all() as { id: string; kind: string }[];
expect(operations.filter(operation => operation.kind === 'instance.create')).toHaveLength(10);
const removals = operations.filter(operation => operation.kind === 'instance.remove');
expect(removals).toHaveLength(9);
const { reportHostOperationStatus } = await import('../../src/lib/server/hosts/index.js');
for (const operation of removals) reportHostOperationStatus(hostId, operation.id, { state: 'completed', result: { exists: false } });
await processProvisioning();
const active = db().prepare('SELECT id,port FROM native_instances WHERE retired_at IS NULL').all() as { id: string; port: number }[];
expect(active).toHaveLength(10);
expect(new Set(active.map(instance => instance.port)).size).toBe(10);
for (const instance of active) expect(db().prepare('SELECT protocol,port FROM ec_host_ports WHERE instance_id=? ORDER BY protocol').all(instance.id))
.toEqual([{ protocol: 'tcp', port: instance.port }, { protocol: 'udp', port: instance.port }]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(20);
expect(getCollection(queued.collectionId!)?.servers.filter(server => server.action === 'destroy').map(server => server.state))
.toEqual(Array(9).fill('done'));
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.remove')").all()).toHaveLength(19);
});
it('accepts nine removals and ten unplaced creations before capacity checks or file loading', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { createScript, setScope, deleteScript } = await import('../../src/lib/server/fleet/scripts.js');
const content = Buffer.alloc(8 * 1024 ** 2, 42);
@@ -91,6 +134,10 @@ describe('matching a native collection', () => {
const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
const { POST } = await import('../../src/routes/api/deployment-collections/+server.js');
const { load } = await import('../../src/routes/collections/launches/[id]/+page.server.js');
const { GET: collectionResponse } = await import('../../src/routes/api/deployment-collections/[id]/+server.js');
const { GET: fleetResponse } = await import('../../src/routes/api/servers/+server.js');
const { POST: agentHeartbeat } = await import('../../src/routes/api/agent/heartbeat/+server.js');
const { T5_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
const blueprint = saveBlueprint({ name: 'Match ten T5 lobbies', entries: T5_ZOMBIE_MAPS.filter(map => map.slug !== 'zombietron')
.map(map => ({ game: 't5', mapSlug: map.slug, count: 1 })) });
@@ -102,6 +149,23 @@ describe('matching a native collection', () => {
.get(preview.id) as { bytes: number };
expect(storedPreview.bytes).toBeLessThan(100_000);
const capacity = await import('../../src/lib/server/hosts/capacity.js');
const placement = vi.spyOn(capacity, 'choosePlacement').mockImplementation(() => {
throw new Error('Placement is waiting for fresh host capacity');
});
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
const evaluate = preparation.evaluateNativePlacement;
let placementStarted!: () => void;
const started = new Promise<void>(resolve => { placementStarted = resolve; });
const gate = new Promise<void>(resolve => { releasePlacement = resolve; });
const evaluation = vi.spyOn(preparation, 'evaluateNativePlacement').mockImplementation(async options => {
placementStarted();
await gate;
return evaluate(options);
});
const scripts = await import('../../src/lib/server/fleet/scripts.js');
const fullVersion = vi.spyOn(scripts, 'getVersion');
const submit = (keys?: Record<string, string>) => POST({ request: new Request('http://localhost/api/deployment-collections', {
method: 'POST', body: JSON.stringify({ previewId: preview.id, keys }) }) });
const response = await submit(Object.fromEntries(preview.servers.map(server => [server.name!, `key-${server.name}`])));
@@ -109,21 +173,54 @@ describe('matching a native collection', () => {
expect(response.status).toBe(202);
expect(body).toMatchObject({ ok: true, collectionId: expect.any(String) });
expect(await (await submit()).json()).toEqual(body);
expect(load({ params: { id: body.collectionId } })).toEqual({});
// Exercise the real route and worker: admission must return before
// loading launch payloads or inspecting the nine servers to remove.
expect(nativeRequest).not.toHaveBeenCalled();
expect(placement).not.toHaveBeenCalled();
expect(evaluation).not.toHaveBeenCalled();
expect(fullVersion).not.toHaveBeenCalled();
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]);
for (const table of ['server_launch_previews', 'server_launch_recipes']) {
const rows = db().prepare(`SELECT length(recipe) AS bytes FROM ${table}`).all() as { bytes: number }[];
expect(rows).toHaveLength(10);
expect(rows.every(row => row.bytes < 10_000)).toBe(true);
}
expect(db().prepare('SELECT id FROM native_instances').all()).toHaveLength(9);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toHaveLength(18);
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toHaveLength(10);
expect(db().prepare('SELECT id FROM server_launch_previews').all()).toEqual([]);
const rows = db().prepare('SELECT length(recipe) AS bytes FROM server_launch_recipes').all() as { bytes: number }[];
expect(rows).toHaveLength(10);
expect(rows.every(row => row.bytes < 10_000)).toBe(true);
const collection = getCollection(body.collectionId)!;
expect(collection.collection).toMatchObject({ mode: 'exact', total: 19 });
expect(collection.servers.filter(server => server.action === 'destroy')).toHaveLength(9);
expect(collection.servers.filter(server => server.action === 'create')).toHaveLength(10);
const { nativeDeployment } = await import('../../src/lib/server/backends/native.js');
for (const server of collection.servers.filter(server => server.action === 'create')) {
expect(nativeDeployment(server.id)).toBe(true);
expect(getDeployment(server.id)?.backend).toBe('native');
}
await new Promise<void>(resolve => setImmediate(resolve));
await processProvisioning({ wait: false });
await started;
expect(evaluation).toHaveBeenCalled();
expect(placement).not.toHaveBeenCalled();
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]);
expect(getCollection(body.collectionId)?.collection.total).toBe(19);
const collectionPage = collectionResponse({ params: { id: body.collectionId } });
expect(collectionPage.status).toBe(200);
expect((await collectionPage.json()).servers).toHaveLength(19);
const fleet = fleetResponse();
expect(fleet.status).toBe(200);
expect((await fleet.json()).servers).toHaveLength(9);
const heartbeatResponse = await agentHeartbeat({ request: new Request('https://controller.test/api/agent/heartbeat', {
method: 'POST', headers: { authorization: `Bearer ${agentCredential}` }, body: JSON.stringify({ inventory: heartbeatInventory })
}) });
expect(heartbeatResponse.status).toBe(200);
expect(await heartbeatResponse.json()).toMatchObject({ host: { id: hostId, connected: true }, heartbeatSeconds: 15 });
// All three endpoints complete while the background placement is still paused.
expect(placement).not.toHaveBeenCalled();
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]);
placement.mockRestore(); fullVersion.mockRestore();
releasePlacement!();
await processProvisioning();
const dispatched = db().prepare(`SELECT n.deployment_id,o.id FROM native_provisioning n
JOIN ec_host_operations o ON o.id=n.operation_id WHERE o.kind='instance.create'`).all() as { deployment_id: string; id: string }[];
tests/unit/create-server.test.ts
+1
View File
@@ -481,6 +481,7 @@ describe('server creation map selection', () => {
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', stage: 'Waiting for RCON or fresh telemetry health: timeout' });
for (const map of [null, 'zombie_moon']) {
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' });
if (created.port === null) throw new Error('A provisioned legacy server must have a port.');
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'localhost', port: created.port }, rawReply: '', status: { players: 0, map, gametype: null, roster: [] } });
await processCreations();
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health', finished_at: null, stage: expect.stringContaining('expected zombie_cosmodrome') });
tests/unit/host-availability.test.ts
+104
View File
@@ -0,0 +1,104 @@
import { setImmediate } from 'node:timers/promises';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
let hostId: string;
const GiB = 1024 ** 3;
const epoch = Date.parse('2026-10-05T12:00:00Z');
const inventory = { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * GiB,
cpuCount: 16, cpuUsagePercent: 0, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB,
runtimeImages: ['image'], libraries: [{ id: 't5-library', state: 'ready' }, { id: 't6-library', state: 'ready' }] };
const t5 = { libraryId: 't5-library', image: 'image', automatic: true };
const t6 = { libraryId: 't6-library', image: 'image', automatic: true };
async function settle() { await setImmediate(); await setImmediate(); }
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch);
const hosts = await import('../../src/lib/server/hosts/index.js');
hostId = hosts.createHost({ name: 'Availability host', portMin: 28960, portMax: 28962 }).id;
hosts.enrollHost(hosts.createEnrollment(hostId).token, inventory);
hosts.heartbeat(hostId, inventory);
await settle();
});
afterEach(async () => { vi.restoreAllMocks(); vi.useRealTimers(); await fixture.cleanup(); });
describe('cached launch availability', () => {
it('reads cached estimates without placement, historical scans, or repeated database reads', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const capacity = await import('../../src/lib/server/hosts/capacity.js');
const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js');
const placement = vi.spyOn(capacity, 'choosePlacement');
const prepare = vi.spyOn(db(), 'prepare');
for (let count = 0; count < 100; count++) {
expect(launchAvailability([t5, t6]).availability).toMatchObject({
updatedAt: new Date(epoch).toISOString(), requested: 2, estimatedAvailable: 2, portsAvailable: 3
});
}
expect(prepare).not.toHaveBeenCalled();
expect(placement).not.toHaveBeenCalled();
});
it('shares CPU, memory and ports across games and refreshes measurements on heartbeat', async () => {
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js');
heartbeat(hostId, { ...inventory, memoryAvailable: GiB + 500_000_000 });
const limited = launchAvailability([t5, t6]);
expect(limited.availability).toMatchObject({ requested: 2, estimatedAvailable: 1, portsAvailable: 3,
cpuCoresAvailable: 15.2, memoryBytesAvailable: GiB });
expect(limited.warnings.join(' ')).toContain('1 of 2');
vi.setSystemTime(epoch + 15_000);
heartbeat(hostId, { ...inventory, memoryAvailable: 2 * GiB + 500_000_000 });
expect(launchAvailability([t5, t6]).availability).toMatchObject({ estimatedAvailable: 2,
memoryBytesAvailable: 2 * GiB, updatedAt: new Date(epoch + 15_000).toISOString() });
heartbeat(hostId, { ...inventory, cpuUsagePercent: 100 });
expect(launchAvailability([t5]).availability.estimatedAvailable).toBe(0);
});
it('refreshes port reservations and startup queue snapshots after lifecycle events', async () => {
const { reserveHostPort, releaseHostPorts } = await import('../../src/lib/server/hosts/index.js');
const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js');
const { beginDeployment, deploymentStage } = await import('../../src/lib/server/provisioning/history.js');
const { db } = await import('../../src/lib/server/db/index.js');
const limit = launchAvailability([t5]).availability.startupLimit!;
reserveHostPort(hostId, 'reserved', { port: 28960, protocol: 'udp' });
reserveHostPort(hostId, 'reserved', { port: 28960, protocol: 'tcp' });
await settle();
expect(launchAvailability([t5, t6, t5]).availability).toMatchObject({ portsAvailable: 2, estimatedAvailable: 2 });
const active = beginDeployment('t5_zombie_theater_1', 'create');
const queued = beginDeployment('t6_zombie_tranzit_1', 'create');
db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(queued);
deploymentStage(queued, 'Awaiting placement', { checkpoint: { key: 'queued', label: 'Queued for placement' } });
await settle();
expect(launchAvailability([t5]).availability).toMatchObject({ queued: 1, startupSlots: Math.max(0, limit - 1) });
db().prepare("UPDATE server_deployments SET state='done' WHERE id=?").run(active);
releaseHostPorts('reserved');
await settle();
expect(launchAvailability([t5]).availability).toMatchObject({ portsAvailable: 3, startupSlots: limit });
});
it('stops counting a stale host without presenting old measurements as current capacity', async () => {
const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js');
vi.setSystemTime(epoch + 91_000);
const result = launchAvailability([t5]);
expect(result.availability).toMatchObject({ updatedAt: new Date(epoch).toISOString(), estimatedAvailable: 0, portsAvailable: 0 });
expect(result.warnings.join(' ')).toContain('Waiting for an eligible host');
});
it('keeps heartbeats working while provisioning schema is unavailable and refreshes after initialization', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js');
const errors = vi.spyOn(console, 'error');
db().exec('ALTER TABLE native_provisioning RENAME TO unavailable_native_provisioning');
expect(() => heartbeat(hostId, inventory)).not.toThrow();
expect(launchAvailability([t5]).availability.updatedAt).toBeNull();
await settle();
expect(errors).not.toHaveBeenCalled();
db().exec('ALTER TABLE unavailable_native_provisioning RENAME TO native_provisioning');
heartbeat(hostId, inventory);
expect(launchAvailability([t5]).availability).toMatchObject({ estimatedAvailable: 1, portsAvailable: 3 });
});
});
tests/unit/host-deletion.test.ts
+41
View File
@@ -77,6 +77,9 @@ describe('host deletion', () => {
const queued = hosts.deleteHost(hosts.host.id);
expect(queued.deleted).toBe(false);
expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('queued');
const { nativeLaunchSelection } = await import('../../src/lib/server/provisioning/native.js');
expect(() => nativeLaunchSelection({ hostId: hosts.host.id, libraryId: 'version', image: 'plutainer:test' }, 't5'))
.toThrow('The selected host is being removed');
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
await finishCleanup(hosts);
@@ -120,6 +123,44 @@ describe('host deletion', () => {
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
});
it('blocks an accepted unplaced pinned launch until it is cancelled', async () => {
const hosts = await enrolled();
const { queueNativeCreationIntent } = await import('../../src/lib/server/provisioning/native.js');
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
const { db } = await import('../../src/lib/server/db/index.js');
const queued = queueNativeCreationIntent({ server: 't5_zombie_theater_1', game: 't5', mapSlug: 'theater',
plutoKey: 'saved-key', selection: { hostId: hosts.host.id, libraryId: 'library', image: 'plutainer:test' } });
expect(queued.ok).toBe(true);
if (!queued.ok) throw new Error(queued.message);
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ allowed: false,
blockers: [expect.stringContaining('Cancel the queued server launches pinned to this host')] });
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
expect(hosts.getHost(hosts.host.id)?.decommission).toBeNull();
expect(cancelCreation(queued.deploymentId!)).toBeTruthy();
expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ allowed: true, blockers: [] });
await finishCleanup(hosts);
expect(hosts.getHost(hosts.host.id)).toBeNull();
});
it('does not pin a host for automatic launches or launches assigned elsewhere', async () => {
const hosts = await enrolled();
const other = hosts.createHost({ name: 'Other host' });
const { queueNativeCreationIntent } = await import('../../src/lib/server/provisioning/native.js');
for (const [index, selection] of [
{ hostId: hosts.host.id, automatic: true },
{ hostId: other.id, automatic: false }
].entries()) {
expect(queueNativeCreationIntent({ server: `t5_zombie_theater_${index + 1}`, game: 't5', mapSlug: 'theater',
plutoKey: 'saved-key', selection: { ...selection, libraryId: 'library', image: 'plutainer:test' } }).ok).toBe(true);
}
expect(hosts.hostDeletionStatus(hosts.host.id)).toEqual({ allowed: true, blockers: [] });
expect(hosts.hostDeletionStatus(other.id).allowed).toBe(false);
await finishCleanup(hosts);
expect(hosts.getHost(hosts.host.id)).toBeNull();
});
it('blocks residual port reservations independently of live instances', async () => {
const hosts = await enrolled();
hosts.reserveHostPort(hosts.host.id, 'unresolved-instance');
tests/unit/host-preparation.test.ts
+62
View File
@@ -243,6 +243,68 @@ describe('automatic host preparation', () => {
expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id);
});
it.each([false, true])('prepares queued runtime and library versions after newer publication (automatic placement: %s)', async automatic => {
const old = await publish();
const id = enroll({ runtimeImages: [runtime] });
const other = enroll({ runtimeImages: [runtime] });
const image = 'ghcr.io/example/plutainer:queued-version';
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
for (let index = 0; index < 2; index++) {
expect(queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', libraryId: old.id,
hostId: automatic ? undefined : id, image, plutoKey: 'saved-key' }).ok).toBe(true);
}
const latest = await publish();
const status = preparation.hostPreparationStatus(hosts.getHost(id)!);
expect(status.items.filter(item => item.label === image)).toHaveLength(1);
expect(status.items.filter(item => item.label.includes('Queued version'))).toHaveLength(1);
const otherStatus = preparation.hostPreparationStatus(hosts.getHost(other)!);
expect(otherStatus.items.some(item => item.label === image)).toBe(automatic);
expect(otherStatus.items.some(item => item.label.includes('Queued version'))).toBe(automatic);
expect(operations(id)).toEqual([]);
preparation.reconcileHostPreparation(id);
const pull = operations(id)[0];
expect(pull).toMatchObject({ kind: 'runtime.pull', payload: { image } });
vi.setSystemTime(Date.now() + 1);
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
heartbeat(id, { runtimeImages: [runtime, image] });
preparation.reconcileHostPreparation(id);
const download = operations(id)[0];
expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: old.id, automatic: true } });
preparation.reconcileHostPreparation(id);
expect(hosts.getHostOperation(download.id)?.state).toBe('pending');
expect(operations(id)).toHaveLength(2);
verified(id, old);
vi.setSystemTime(Date.now() + 1);
hosts.reportHostOperation(id, download.id, { state: 'completed' });
heartbeat(id, { runtimeImages: [runtime, image], libraries: [{ id: old.id, state: 'ready' }] });
preparation.reconcileHostPreparation(id);
expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } });
});
it('retains an undelivered library download for an accepted intent until that intent is cancelled', async () => {
const old = await publish();
const id = enroll({ runtimeImages: [runtime] });
preparation.reconcileHostPreparation(id);
const download = operations(id)[0];
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const launch = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', libraryId: old.id,
hostId: id, image: runtime, plutoKey: 'saved-key' });
expect(launch.ok).toBe(true);
if (!launch.ok) throw new Error(launch.message);
const latest = await publish();
preparation.reconcileHostPreparation(id);
expect(hosts.getHostOperation(download.id)?.state).toBe('pending');
expect(operations(id)).toHaveLength(1);
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
expect(cancelCreation(launch.deploymentId)).toEqual(expect.any(String));
preparation.reconcileHostPreparation(id);
expect(hosts.getHostOperation(download.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('Superseded') });
expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } });
});
it('finishes a delivered download before following a newer upload under a new operation ID', async () => {
const old = await publish();
const id = enroll({ runtimeImages: [runtime] });
tests/unit/native-creation-intents.test.ts
+197
View File
@@ -0,0 +1,197 @@
import Database from 'better-sqlite3';
import { createHash } from 'node:crypto';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import schema from '../../src/lib/server/db/schema.sql?raw';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const server = 't5_zombie_theater_1';
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', '');
vi.stubEnv('RCON_PASSWORD', '');
});
afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); vi.unstubAllEnvs(); });
async function intent() {
const { db } = await import('../../src/lib/server/db/index.js');
const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js');
ensureHostSchema(db());
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
const id = beginDeployment(server, 'create');
db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id);
db().prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)')
.run(id, JSON.stringify({ automatic: true, libraryId: 'library', image: 'plutainer:test' }));
db().prepare(`INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map)
VALUES(?,?,NULL,'zombie','theater','zombie_theater')`).run(id, server);
return id;
}
describe('native creation intents before placement', () => {
it('retains its identity and native history across restart without falling into legacy provisioning', async () => {
const id = await intent();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const { getDeployment, interruptDeployments } = await import('../../src/lib/server/provisioning/history.js');
const { nativeDeployment, unplacedNativeCreation } = await import('../../src/lib/server/backends/native.js');
const { nextLobbyNumber, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
expect(nativeDeployment(id)).toBe(true);
expect(unplacedNativeCreation(server)).toBe(id);
expect(nextLobbyNumber('t5_zombie_theater')).toBe(2);
expect(requestLifecycle(server, 'stop')).toMatchObject({ ok: false, status: 409 });
closeDb();
interruptDeployments();
await processCreations();
expect(getDeployment(id)).toMatchObject({ backend: 'native', canCancelCreation: true, agentOperation: null,
deployment: { state: 'queued', server_exists: 0 } });
expect(db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(id)).toEqual({ port: null });
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT deployment_id FROM native_provisioning').all()).toEqual([]);
});
it('cancels unplaced work locally once, retains keys, and frees its logical server name', async () => {
const id = await intent();
const { db } = await import('../../src/lib/server/db/index.js');
const { saveServerKey, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
const { nativeDeployment, unplacedNativeCreation } = await import('../../src/lib/server/backends/native.js');
const { nextLobbyNumber } = await import('../../src/lib/server/provisioning/instances.js');
saveServerKey(server, 'saved-key');
expect(canCancelCreation(id)).toBe(true);
const cleanup = cancelCreation(id)!;
expect(cleanup).toEqual(expect.any(String));
expect(cancelCreation(id)).toBe(cleanup);
expect(canCancelCreation(id)).toBe(false);
expect(nativeDeployment(cleanup)).toBe(true);
expect(getDeployment(id)).toMatchObject({ backend: 'native', cancellation: { state: 'done' }, deployment: { state: 'cancelled' } });
expect(getDeployment(cleanup)).toMatchObject({ backend: 'native', deployment: { state: 'done', server_deleted_at: expect.any(String) } });
expect(unplacedNativeCreation(server)).toBeNull();
expect(nextLobbyNumber('t5_zombie_theater')).toBe(1);
expect(getServerKey(server)).toBe('saved-key');
expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual([]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
});
it('will not cancel an unplaced historical intent against a replacement server', async () => {
const id = await intent();
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("INSERT INTO servers(name,komodo_name) VALUES(?,'replacement')").run(server);
const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
expect(canCancelCreation(id)).toBe(false);
expect(cancelCreation(id)).toBeNull();
expect(db().prepare('SELECT name FROM servers WHERE name=?').get(server)).toEqual({ name: server });
expect(db().prepare('SELECT cleanup_id FROM native_creation_cancellations').all()).toEqual([]);
});
it.each([false, true])('does not adopt a replacement instance during placement (previously bound: %s)', async previouslyBound => {
const id = await intent();
const { db } = await import('../../src/lib/server/db/index.js');
const { createHost } = await import('../../src/lib/server/hosts/index.js');
const { captureRecipePlan, recipeStorage } = await import('../../src/lib/server/fleet/launch-recipes.js');
const hostId = createHost({ name: 'Replacement host' }).id;
const insert = db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at,retired_at)
VALUES(?,?,?,'library','plutainer:test',28960,?,?)`);
if (previouslyBound) {
insert.run('original-instance', server, hostId, '2025-01-01', '2025-01-02');
db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(id, 'original-instance');
}
insert.run('replacement-instance', server, hostId, '2025-01-02', null);
const before = db().prepare('SELECT * FROM native_instances ORDER BY id').all();
const stored = recipeStorage(captureRecipePlan('t5', 'theater'));
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)')
.run(id, stored.recipe, stored.recipeSummary);
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
const placement = vi.spyOn(preparation, 'evaluateNativePlacement').mockRejectedValue(new Error('Unexpected placement'));
const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
await processNativeProvisioning(() => null);
expect(placement).not.toHaveBeenCalled();
expect(db().prepare('SELECT state,error FROM server_deployments WHERE id=?').get(id))
.toEqual({ state: 'failed', error: 'Original instance was retired or replaced; no command was sent.' });
expect(db().prepare('SELECT * FROM native_instances ORDER BY id').all()).toEqual(before);
expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual([]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
expect(db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(id)).toEqual({ port: null });
});
it('starts work on a ready host when an older pinned intent is waiting for an offline host', async () => {
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
const image = 'plutainer:test';
const hosts = await import('../../src/lib/server/hosts/index.js');
const offline = hosts.createHost({ name: 'Offline host' });
const ready = hosts.createHost({ name: 'Ready host' });
const inventory = { protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
cpuCount: 16, cpuUsagePercent: 5, memoryTotal: 32 * 1024 ** 3, memoryAvailable: 30 * 1024 ** 3,
storageAvailable: 100_000_000_000, runtimeImages: [image] };
hosts.enrollHost(hosts.createEnrollment(ready.id).token, inventory);
const libraries = await import('../../src/lib/server/game-files/index.js');
const imported = libraries.createLibraryImport({ hostId: ready.id, game: 't5' });
const bytes = Buffer.from('verified base game');
const file = { path: 'zone/base.ff', size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') };
libraries.setImportManifest(ready.id, imported.id, [file]);
await libraries.uploadImportFile(ready.id, imported.id, file.path, new Response(bytes).body);
const library = await libraries.publishImport(ready.id, imported.id);
libraries.updateHostLibraryCaches(ready.id, [{ libraryVersionId: library.id, status: 'ready', manifestSha256: library.manifestSha256 }]);
hosts.heartbeat(ready.id, { ...inventory, libraries: [{ id: library.id, state: 'ready' }] });
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
const { queueNativeCreation, processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', libraryId: library.id, image, plutoKey: 'saved-key' };
const blocked = queueNativeCreation({ ...input, hostId: offline.id });
const available = queueNativeCreation({ ...input, hostId: ready.id });
expect(blocked.ok && available.ok).toBe(true);
if (!blocked.ok || !available.ok) throw new Error('Could not queue launch fixtures.');
await processNativeProvisioning(() => null);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(blocked.deploymentId)).toEqual({ state: 'queued' });
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(available.deploymentId)).toEqual({ state: 'running' });
expect(db().prepare("SELECT host_id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([{ host_id: ready.id }]);
expect(db().prepare('SELECT server,host_id FROM native_instances WHERE retired_at IS NULL').all())
.toEqual([{ server: available.name, host_id: ready.id }]);
expect(db().prepare('SELECT port FROM server_creation_jobs WHERE deployment_id=?').get(blocked.deploymentId)).toEqual({ port: null });
});
it('handles exact-match removal of an unplaced launch as local cancellation', async () => {
const id = await intent();
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
const { db } = await import('../../src/lib/server/db/index.js');
const removal = destroyServer(server, 'blueprint-exact');
expect(removal).toMatchObject({ ok: true, deploymentId: expect.any(String) });
expect(getDeployment(id)?.deployment.state).toBe('cancelled');
expect(getDeployment(removal.deploymentId!)?.deployment.state).toBe('done');
expect(db().prepare('SELECT id FROM ec_host_operations').all()).toEqual([]);
});
it('migrates existing port reservations without losing rows, references, indexes, or triggers', async () => {
const legacy = new Database(fixture.dbPath);
try {
legacy.exec(schema.replace(/CREATE TABLE IF NOT EXISTS server_creation_jobs \([\s\S]*?\n\);/,
table => table.replace(' port INTEGER,', ' port INTEGER NOT NULL,')));
legacy.exec(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
VALUES('existing','old-server','create','queued','Queued','2025-01-01','fixture');
INSERT INTO server_creation_jobs(rowid,deployment_id,server,port,game_mode,map_slug,expected_map)
VALUES(8,'existing','old-server',28960,'zombie','theater','zombie_theater');
CREATE INDEX fixture_creation_port ON server_creation_jobs(port);
CREATE TABLE fixture_creation_reference(id TEXT REFERENCES server_creation_jobs(deployment_id));
INSERT INTO fixture_creation_reference(id) VALUES('existing');
CREATE TABLE fixture_creation_audit(port INTEGER);
CREATE TRIGGER fixture_creation_update AFTER UPDATE OF port ON server_creation_jobs
BEGIN INSERT INTO fixture_creation_audit(port) VALUES(NEW.port); END;`);
} finally { legacy.close(); }
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
for (let open = 0; open < 2; open++) {
const handle = db();
expect(handle.prepare('SELECT rowid,deployment_id,port FROM server_creation_jobs').get())
.toEqual({ rowid: 8, deployment_id: 'existing', port: 28960 });
expect(handle.prepare('SELECT id FROM fixture_creation_reference').get()).toEqual({ id: 'existing' });
expect(handle.prepare("SELECT name FROM sqlite_master WHERE name IN ('fixture_creation_port','fixture_creation_update')").all()).toHaveLength(2);
expect(handle.pragma('foreign_key_check')).toEqual([]);
closeDb();
}
db().prepare('UPDATE server_creation_jobs SET port=NULL WHERE deployment_id=?').run('existing');
expect(db().prepare('SELECT port FROM fixture_creation_audit').all()).toEqual([{ port: null }]);
});
});
tests/unit/native-launch-preparation.test.ts
+27
View File
@@ -83,6 +83,33 @@ async function pausePreparation() {
}
describe('asynchronous native launch preparation', () => {
it('does not place an unsubmitted intent cancelled while host ranking is pending', async () => {
const job = await queue();
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
const evaluate = preparation.evaluateNativePlacement;
let started!: () => void;
let release!: () => void;
const ready = new Promise<void>(resolve => { started = resolve; });
const gate = new Promise<void>(resolve => { release = resolve; });
releases.push(release);
vi.spyOn(preparation, 'evaluateNativePlacement').mockImplementationOnce(async input => {
const result = await evaluate(input);
started(); await gate;
return result;
});
const pass = runPass();
await ready;
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
expect(nativeInstance(job.name)).toBeNull();
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String));
release(); await pass;
expect(nativeInstance(job.name)).toBeNull();
expect(await operationFor(job.deploymentId)).toBeUndefined();
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
});
it('keeps controller timers and launch details responsive while preparing multiple large plugins', async () => {
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { db } = await import('../../src/lib/server/db/index.js');
tests/unit/native-provisioning.test.ts
+141 -34
View File
@@ -43,10 +43,11 @@ async function prepareLibrary(game: 't5' | 't6' = 't5') {
const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory());
return ready;
}
async function queue(game: 't5' | 't6' = 't5') {
async function queue(game: 't5' | 't6' = 't5', placement: 'pinned' | 'automatic' = 'pinned') {
await prepareLibrary(game); remote.map = game === 't5' ? 'zombie_theater' : 'zm_transit';
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
const result = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town', hostId, libraryId, image, plutoKey: 'saved-key' });
const result = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town',
hostId: placement === 'pinned' ? hostId : undefined, libraryId, image, plutoKey: 'saved-key' });
if (!result.ok) throw new Error(result.message);
return result;
}
@@ -73,6 +74,44 @@ async function detail(id: string) {
}
describe('native instance provisioning', () => {
it('recognizes an unplaced native intent and resumes it once after a controller restart', async () => {
const job = await queue();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const { nativeDeployment, nativeInstance } = await import('../../src/lib/server/backends/native.js');
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
expect(job.port).toBeNull();
expect(nativeDeployment(job.deploymentId)).toBe(true);
expect(nativeInstance(job.name)).toBeNull();
expect(getDeployment(job.deploymentId)).toMatchObject({ backend: 'native', canCancelCreation: true });
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([{ deployment_id: job.deploymentId }]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
closeDb(); vi.resetModules();
const backend = await import('../../src/lib/server/backends/native.js');
expect(backend.nativeDeployment(job.deploymentId)).toBe(true);
await advance();
const original = await operationFor(job.deploymentId);
expect(original).toMatchObject({ kind: 'instance.create', payload: { name: job.name, port: 28960 } });
await advance();
expect((await operationFor(job.deploymentId))?.id).toBe(original!.id);
const restartedDb = (await import('../../src/lib/server/db/index.js')).db();
expect(restartedDb.prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
expect(restartedDb.prepare('SELECT protocol FROM ec_host_ports ORDER BY protocol').all()).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]);
});
it('still rejects missing keys and stale lobby identities before creating an intent', async () => {
await prepareLibrary();
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', hostId, libraryId, image };
expect(queueNativeCreation(input)).toMatchObject({ ok: false, message: expect.stringContaining('server key') });
expect(queueNativeCreation({ ...input, plutoKey: 'saved-key', expectedName: 't5_zombie_theater_2' }))
.toMatchObject({ ok: false });
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([]);
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
});
it('accepts cancellation through the provisioning API and rejects an already completed creation', async () => {
const job = await queue();
const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js');
@@ -341,11 +380,30 @@ describe('native instance provisioning', () => {
expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ count: 0 });
});
it('keeps a queued server pinned to its accepted files when newer game files arrive', async () => {
const job = await queue();
it.each(['pinned', 'automatic'] as const)('keeps an unplaced %s launch pinned to its accepted files during cleanup', async placement => {
const job = await queue('t5', placement);
const acceptedId = libraryId;
const latest = await prepareLibrary('t5');
expect(latest.id).not.toBe(acceptedId);
const { db } = await import('../../src/lib/server/db/index.js');
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
const { reconcileHostLibraryCleanup, cleanupControllerLibraries, getLibraryVersion, updateHostLibraryCaches }
= await import('../../src/lib/server/game-files/index.js');
const accepted = getLibraryVersion(acceptedId)!;
heartbeat(hostId, { ...inventory(), capabilities: ['library.remove'] });
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
reconcileHostLibraryCleanup(hostId);
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=?")
.all(acceptedId)).toEqual([]);
// The unplaced intent also protects the controller's copy when every
// host reports it absent; its accepted version remains downloadable.
updateHostLibraryCaches(hostId, [{ libraryVersionId: acceptedId, status: 'missing' }]);
await cleanupControllerLibraries();
expect(db().prepare('SELECT version_id FROM game_file_cleanup WHERE version_id=?').get(acceptedId)).toBeUndefined();
const { stat } = await import('node:fs/promises');
expect((await stat(join(fixture.root, 'libraries', 't5', acceptedId, 'zone/base.ff'))).isFile()).toBe(true);
updateHostLibraryCaches(hostId, [{ libraryVersionId: acceptedId, status: 'ready', manifestSha256: accepted.manifestSha256 }]);
await advance();
expect((await operationFor(job.deploymentId))?.payload.libraryId).toBe(acceptedId);
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
@@ -355,7 +413,8 @@ describe('native instance provisioning', () => {
it.each(['t5', 't6'] as const)('queues and verifies a %s deployment without Komodo', async game => {
const job = await queue(game); await advance();
const operation = await operationFor(job.deploymentId);
expect(operation).toMatchObject({ kind: 'instance.create', hostId, generation: 1, state: 'pending', payload: { game, libraryId, image, port: job.port } });
expect(job.port).toBeNull();
expect(operation).toMatchObject({ kind: 'instance.create', hostId, generation: 1, state: 'pending', payload: { game, libraryId, image, port: 28960 } });
expect(operation?.payload.files).toEqual(expect.arrayContaining([expect.objectContaining({ path: 'configs/electric_cherry_map.cfg' })]));
await acknowledge(job.deploymentId);
expect((await detail(job.deploymentId)).state).toBe('done');
@@ -534,17 +593,22 @@ describe('native instance provisioning', () => {
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe(createHash('sha256').update(after).digest('hex'));
});
it('atomically launches mixed-game collections with per-host port reservations', async () => {
it('atomically queues mixed-game collections and reserves per-host ports when dispatched', async () => {
const t5 = await prepareLibrary('t5'); const t6 = await prepareLibrary('t6');
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const blueprint = saveBlueprint({ name: 'Both games', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }, { game: 't6', mapSlug: 'town', count: 1 }] });
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const preview = previewCollection(blueprint.id, 'fill', { hostId, libraryIds: { t5: t5.id, t6: t6.id }, image });
expect(preview.ready).toBe(true);
expect(preview.servers.map(server => server.port)).toEqual([28960, 28961, 28962]);
expect(preview.servers.map(server => server.port)).toEqual([null, null, null]);
const queued = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) });
expect(queueCollection({ previewId: preview.id }).collectionId).toBe(queued.collectionId);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 });
await advance();
const instances = db().prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port').all();
expect(instances).toEqual([{ host_id: hostId, library_id: t5.id, port: 28960 }, { host_id: hostId, library_id: t5.id, port: 28961 }, { host_id: hostId, library_id: t6.id, port: 28962 }]);
const { createHost, createEnrollment, enrollHost, hostPortAvailable } = await import('../../src/lib/server/hosts/index.js');
@@ -554,13 +618,15 @@ describe('native instance provisioning', () => {
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'moon', hostId: secondHost.id, libraryId: t5.id, image, plutoKey: 'second-host-key' });
expect(second.ok).toBe(true); if (!second.ok) throw new Error(second.message);
expect(second.port).toBe(28960);
expect(second.port).toBeNull();
await advance();
expect((await operationFor(second.deploymentId))?.payload.port).toBe(28960);
expect(hostPortAvailable(hostId, 28960)).toBe(false);
expect(hostPortAvailable(secondHost.id, 28960)).toBe(false);
expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ count: 8 });
});
it('reads remote instance files through the agent including missing files and epoch timestamps', async () => {
const job = await queue();
const job = await queue(); await advance();
const backend = await import('../../src/lib/server/backends/native.js');
const remoteRead = vi.mocked(backend.nativeRequest);
remoteRead.mockImplementation(async (_instance, kind, payload) => {
@@ -622,17 +688,17 @@ describe('native instance provisioning', () => {
const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
expect(getServerScript(job.name, 'hud.gsc')).toBeUndefined();
});
it('cancels a queued native launch locally and releases both ports without submitting a create', async () => {
it('cancels an unplaced native launch locally without reserving ports or submitting a create', async () => {
const job = await queue();
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
const instance = nativeInstance(job.name)!;
expect(nativeInstance(job.name)).toBeNull();
const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
const cancelled = destroyServer(job.name); expect(cancelled.ok).toBe(true);
await advance();
const { db } = await import('../../src/lib/server/db/index.js');
expect((await detail(job.deploymentId)).state).toBe('cancelled');
expect(nativeInstance(job.name)).toBeNull();
expect(db().prepare('SELECT * FROM ec_host_ports WHERE instance_id=?').all(instance.id)).toEqual([]);
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind LIKE 'instance.%'").get()).toEqual({ count: 0 });
const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js');
expect(getLibraryVersion(libraryId)?.state).toBe('ready');
@@ -661,7 +727,8 @@ describe('automatic capacity admission and placement', () => {
const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
const result = queueCreateServer(input);
expect(result.ok).toBe(true); if (!result.ok) return;
expect(nativeInstance(result.name)?.host_id).toBe(hostId); // Fast first fit.
expect(nativeInstance(result.name)).toBeNull();
expect(result.port).toBeNull();
await advance();
expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id);
expect(nativeInstance(result.name)).toMatchObject({ host_id: other.id, public_address: 'second.example' });
@@ -689,13 +756,18 @@ describe('automatic capacity admission and placement', () => {
it.each([
{ cpuUsagePercent: undefined }, { memoryAvailable: undefined }, { cpuCount: 0 },
{ cpuUsagePercent: 99 }, { memoryAvailable: 1024 ** 3 }
])('rejects admission with unavailable resources: %j', async overrides => {
])('accepts an unplaced job and waits for unavailable resources: %j', async overrides => {
await prepareLibrary();
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), ...overrides });
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const { db } = await import('../../src/lib/server/db/index.js');
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict', message: expect.stringContaining('capacity') });
const queued = queueNativeCreation(input);
expect(queued).toMatchObject({ ok: true, port: null });
if (!queued.ok) throw new Error(queued.message);
await advance();
expect(await operationFor(queued.deploymentId)).toBeNull();
expect((await detail(queued.deploymentId)).stage).toContain('capacity');
expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
});
@@ -709,22 +781,28 @@ describe('automatic capacity admission and placement', () => {
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId);
const result = queueNativeCreation({ ...input, previewId: preview.id });
expect(result).toMatchObject({ ok: true, name: preview.name, port: 30000 });
expect(result).toMatchObject({ ok: true, name: preview.name, port: null });
expect(queueNativeCreation({ ...input, previewId: preview.id })).toEqual(result);
if (!result.ok) throw new Error(result.message);
await advance();
expect((await operationFor(result.deploymentId))?.payload.port).toBe(30000);
});
it('rejects an entire collection when capacity shrinks after its preview', async () => {
it('queues the whole collection and waits when capacity shrinks after its preview', async () => {
await prepareLibrary();
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
const preview = previewCollection(blueprint.id);
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
expect(() => queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) })).toThrow('capacity');
heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 });
const result = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) });
expect(result).toMatchObject({ ok: true, collectionId: expect.any(String) });
await advance();
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toHaveLength(1);
expect(db().prepare('SELECT * FROM native_creation_intents').all()).toHaveLength(2);
});
it.each([
{ resource: 'CPU', measurements: { cpuCount: 4, cpuUsagePercent: 82.5 }, over: { cpuUsagePercent: 82.51 } },
@@ -742,15 +820,19 @@ describe('automatic capacity admission and placement', () => {
expect(() => choosePlacement(options)).toThrow('capacity');
expect(() => choosePlacement({ ...options, thorough: true })).toThrow('capacity');
});
it('uses calibrated footprints to admit more instances than the cold-start allowance', async () => {
it('dispatches a waiting instance after calibration lowers the cold-start allowance', async () => {
const first = await queue();
await advance(); await acknowledge(first.deploymentId);
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
const { recordInstanceResources, capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const measured = { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 };
const measured = { ...inventory(), cpuCount: 1, cpuUsagePercent: 50 };
heartbeat(hostId, measured);
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
const next = queueNativeCreation(input);
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
await advance();
expect(await operationFor(next.deploymentId)).toBeNull();
const instance = nativeInstance(first.name)!;
for (let minute = 0; minute < 25; minute++) {
recordInstanceResources(hostId, { ...measured, instanceResources: [{ instanceId: instance.id, generation: 1,
@@ -758,7 +840,8 @@ describe('automatic capacity admission and placement', () => {
new Date(Date.now() - (25 - minute) * 60_000).toISOString());
}
expect(capacityCalibration(hostId).startup.source).toBe('host');
expect(queueNativeCreation(input)).toMatchObject({ ok: true });
await advance();
expect(await operationFor(next.deploymentId)).not.toBeNull();
});
it('queues a collection at running demand and releases startup reservations as live usage arrives', async () => {
await prepareLibrary();
@@ -791,7 +874,7 @@ describe('automatic capacity admission and placement', () => {
await advance();
expect(creates()).toHaveLength(2); // Its actual usage replaces the peak even before health verification.
});
it('charges queued reservations and releases them on cancellation', async () => {
it('reserves queued lobby identities without host resources and releases them on cancellation', async () => {
await prepareLibrary();
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
@@ -799,9 +882,13 @@ describe('automatic capacity admission and placement', () => {
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
const first = queueNativeCreation(input);
expect(first.ok).toBe(true); if (!first.ok) return;
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
const second = queueNativeCreation(input);
expect(second).toMatchObject({ ok: true, name: 't5_zombie_theater_2', port: null });
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
cancelCreation(first.deploymentId);
expect(queueNativeCreation(input)).toMatchObject({ ok: true });
expect(queueNativeCreation(input)).toMatchObject({ ok: true, name: first.name, port: null });
});
it.each(['memory', 'cpu'] as const)('dispatches immediately after %s recovers instead of waiting for five-minute history', async resource => {
await prepareLibrary();
@@ -846,7 +933,10 @@ describe('automatic capacity admission and placement', () => {
// even though the next host heartbeat has not observed it yet.
await advance();
expect(db().prepare("SELECT * FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
const later = queueNativeCreation(input);
expect(later.ok).toBe(true); if (!later.ok) throw new Error(later.message);
await advance();
expect(await operationFor(later.deploymentId)).toBeNull();
});
it('does not charge a starting container twice when its usage is already in the heartbeat', async () => {
await prepareLibrary();
@@ -873,6 +963,8 @@ describe('automatic capacity admission and placement', () => {
});
it.each(['missing', 'old-generation'] as const)('keeps full reservations for a submitted launch with %s measurements', async evidence => {
await prepareLibrary();
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
@@ -885,8 +977,11 @@ describe('automatic capacity admission and placement', () => {
heartbeat(hostId, { ...inventory(), memoryTotal: 3 * GiB, memoryAvailable: 2 * GiB,
instanceResources: evidence === 'missing' ? [] : [{ instanceId: instance.id, generation: instance.generation + 1,
uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict',
message: expect.stringContaining('3.00 GiB projected, 2.53 GiB usable') });
const next = queueNativeCreation(input);
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
await advance();
expect(await operationFor(next.deploymentId)).toBeNull();
expect((await detail(next.deploymentId)).stage).toContain('3.00 GiB projected, 2.53 GiB usable');
});
it('waits when capacity disappears after admission without sending a create command', async () => {
await prepareLibrary();
@@ -931,11 +1026,16 @@ describe('automatic capacity admission and placement', () => {
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
const result = queueNativeCreation(input);
expect(result.ok).toBe(true); if (!result.ok) return;
expect(nativeInstance(result.name)).toBeNull();
await advance();
expect(nativeInstance(result.name)?.host_id).toBe(other.id);
updateHost(other.id, { mode: 'draining' });
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
const next = queueNativeCreation(input);
expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
await advance();
expect(await operationFor(next.deploymentId)).toBeNull();
});
it('spreads collection admission across hosts, allowing the same port on different hosts', async () => {
it('spreads collection dispatch across hosts, allowing the same port on different hosts', async () => {
await prepareLibrary(); await secondHost({ cpuCount: 1, cpuUsagePercent: 0 });
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
@@ -944,10 +1044,17 @@ describe('automatic capacity admission and placement', () => {
const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
const preview = previewCollection(blueprint.id);
expect(preview.ready).toBe(true);
expect(preview.servers.map(server => server.port)).toEqual([28960, 28960]);
expect(new Set(preview.servers.map(server => server.native?.hostId)).size).toBe(2);
expect(preview.servers.map(server => server.port)).toEqual([null, null]);
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) }).ok).toBe(true);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
await advance();
// Concurrent ranking can select the same host before the first
// reservation commits. The held job retries against the new usage.
await advance();
expect(db().prepare('SELECT DISTINCT host_id FROM native_instances').all()).toHaveLength(2);
expect(db().prepare('SELECT port FROM native_instances').all()).toEqual([{ port: 28960 }, { port: 28960 }]);
});
});