Restrict launches to prepared runtimes (Refs #93)
CI / checks (push) Successful in 4m53s

This commit is contained in:
2026-10-09 17:15:15 -04:00
parent f0afd6de67
commit 596b160be3
36 changed files with 347 additions and 116 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ const inventory = {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100 * 1024 ** 3,
runtimeImages: ['plutainer:test'],
preparedRuntimeImages: ['plutainer:test'],
libraries: [{ id: 'test-library', state: 'ready' }],
capabilities: ['instance.cancel-create', 'operations.health-lane']
};
+1 -1
View File
@@ -18,7 +18,7 @@ const inventory = {
cpuUsagePercent: 0,
memoryTotal: 64 * GiB,
memoryAvailable: 60 * GiB,
runtimeImages: [image],
preparedRuntimeImages: [image],
libraries: [
{ id: 't5-version', state: 'ready' },
{ id: 't6-version', state: 'ready' }
@@ -35,7 +35,7 @@ beforeEach(async () => {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 1_000_000_000_000,
runtimeImages: [image]
preparedRuntimeImages: [image]
};
enrollHost(createEnrollment(hostId).token, inventory);
const { createLibraryImport, updateHostLibraryCaches } =
@@ -47,7 +47,7 @@ beforeEach(async () => {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 1_000_000_000_000,
runtimeImages: [image]
preparedRuntimeImages: [image]
};
agentCredential = enrollHost(createEnrollment(hostId).token, inventory).credential;
const { createLibraryImport, updateHostLibraryCaches } =
@@ -33,7 +33,7 @@ beforeEach(async () => {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100 * 1024 ** 3,
runtimeImages: [runtime]
preparedRuntimeImages: [runtime]
});
version = await publish();
});
+1 -1
View File
@@ -8,7 +8,7 @@ const inventory = {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100 * 1024 ** 3,
runtimeImages: ['image'],
preparedRuntimeImages: ['image'],
libraries: [{ id: 'library', state: 'ready' }]
};
const lane: HostAgentLane = {
+27 -1
View File
@@ -15,7 +15,7 @@ const inventory = {
cpuUsagePercent: 0,
memoryTotal: 64 * GiB,
memoryAvailable: 60 * GiB,
runtimeImages: ['image'],
preparedRuntimeImages: ['image'],
libraries: [
{ id: 't5-library', state: 'ready' },
{ id: 't6-library', state: 'ready' }
@@ -46,6 +46,32 @@ afterEach(async () => {
});
describe('cached launch availability', () => {
it.each([{ preparedRuntimeImages: [] }, { preparedRuntimeImages: ['another:image'] }])(
'excludes a cached image until it is prepared ($preparedRuntimeImages)',
async ({ preparedRuntimeImages }) => {
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
const { launchAvailability } =
await import('../../src/lib/server/hosts/availability.js');
heartbeat(hostId, {
...inventory,
runtimeImages: ['image'],
preparedRuntimeImages
});
const unavailable = launchAvailability([t5]);
expect(unavailable.availability).toMatchObject({
estimatedAvailable: 0,
portsAvailable: 0
});
expect(unavailable.warnings.join(' ')).toContain('Waiting for an eligible host');
heartbeat(hostId, { ...inventory, runtimeImages: ['image'] });
expect(launchAvailability([t5]).availability).toMatchObject({
estimatedAvailable: 1,
portsAvailable: 3
});
}
);
it('reads cached estimates without placement, historical scans, or repeated database reads', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const capacity = await import('../../src/lib/server/hosts/capacity.js');
+1 -1
View File
@@ -14,7 +14,7 @@ const inventory = {
protocolVersion: 1,
directoriesWritable: true,
storageAvailable: 100 * GiB,
runtimeImages: ['image'],
preparedRuntimeImages: ['image'],
libraries: [{ id: 'library', state: 'ready' }]
};
const sample = {
@@ -9,7 +9,7 @@ const inventory = {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100 * 1024 ** 3,
runtimeImages: ['plutainer:test'],
preparedRuntimeImages: ['plutainer:test'],
libraries: [{ id: 'library', state: 'ready' }]
};
let fixture: Fixture;
+55 -42
View File
@@ -18,7 +18,7 @@ const inventory: HostInventory = {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100 * 1024 ** 3,
runtimeImages: [],
preparedRuntimeImages: [],
libraries: [],
capabilities: ['host.decommission']
};
@@ -123,7 +123,7 @@ describe('automatic host preparation', () => {
const t5 = await publish('t5');
const t6 = await publish('t6');
const readyInventory = {
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const })),
capabilities: [
...inventory.capabilities!,
@@ -195,7 +195,7 @@ describe('automatic host preparation', () => {
it('requires both games when enabling even if one library is already cached', async () => {
const version = await publish('t5');
const id = enroll({
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
libraries: [{ id: version.id, state: 'ready' }]
});
verified(id, version);
@@ -212,7 +212,7 @@ describe('automatic host preparation', () => {
const t5 = await publish('t5');
const t6 = await publish('t6');
const readyInventory = {
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const }))
};
const id = enroll(readyInventory);
@@ -310,7 +310,11 @@ describe('automatic host preparation', () => {
expect(operations(host.id)).toHaveLength(1);
vi.setSystemTime(Date.now() + 1);
const preparedInventory = { ...inventory, runtimeImages: [runtime], libraryCaches: [] };
const preparedInventory = {
...inventory,
preparedRuntimeImages: [runtime],
libraryCaches: []
};
const runtimeHeartbeat = await reportHeartbeat({
request: request('heartbeat', { inventory: preparedInventory })
});
@@ -369,7 +373,7 @@ describe('automatic host preparation', () => {
it('presents preparation without queuing work and prepares the configured runtime even when another image exists', async () => {
const version = await publish();
const id = enroll({ runtimeImages: ['another:runtime'] });
const id = enroll({ preparedRuntimeImages: ['another:runtime'] });
const host = hosts.getHost(id)!;
const status = preparation.hostPreparationStatus(host);
expect(status.items.map((item) => item.kind)).toEqual(['runtime', 'library']);
@@ -406,7 +410,7 @@ describe('automatic host preparation', () => {
preparation.reconcileHostPreparation(id);
expect(operations(id)).toHaveLength(1);
heartbeat(id, { runtimeImages: [runtime] });
heartbeat(id, { preparedRuntimeImages: [runtime] });
preparation.reconcileHostPreparation(id);
expect(operations(id)).toHaveLength(2);
const download = operations(id)[0];
@@ -417,7 +421,7 @@ describe('automatic host preparation', () => {
verified(id, version);
hosts.reportHostOperation(id, download.id, { state: 'completed' });
heartbeat(id, {
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
libraries: [{ id: version.id, state: 'ready' }]
});
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
@@ -432,7 +436,7 @@ describe('automatic host preparation', () => {
game: 't5',
files: [{ path: 'zone/game.ff', size: content.length, lastModified: 0 }]
});
const id = enroll({ runtimeImages: [runtime] });
const id = enroll({ preparedRuntimeImages: [runtime] });
const expected = new Map([latest, t6].map((version) => [version.id, version]));
const downloaded: string[] = [];
for (let index = 0; index < expected.size; index++) {
@@ -452,7 +456,7 @@ describe('automatic host preparation', () => {
preparation.reconcileHostPreparation(id);
expect(operations(id)).toHaveLength(index + 1);
heartbeat(id, {
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
libraries: downloaded.map((id) => ({ id, state: 'ready' }))
});
}
@@ -462,12 +466,15 @@ describe('automatic host preparation', () => {
});
it('picks up publication after enrollment and keeps verified historical versions intact', async () => {
const id = enroll({ runtimeImages: [runtime] });
const id = enroll({ preparedRuntimeImages: [runtime] });
preparation.reconcileHostPreparation(id);
expect(operations(id)).toEqual([]);
const old = await publish();
verified(id, old);
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
heartbeat(id, {
preparedRuntimeImages: [runtime],
libraries: [{ id: old.id, state: 'ready' }]
});
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
const latest = await publish();
preparation.reconcileHostPreparation(id);
@@ -479,7 +486,7 @@ describe('automatic host preparation', () => {
it('replaces an undelivered automatic download when newer files finish uploading', async () => {
const old = await publish();
const id = enroll({ runtimeImages: [runtime] });
const id = enroll({ preparedRuntimeImages: [runtime] });
preparation.reconcileHostPreparation(id);
const first = operations(id)[0];
expect(first.payload).toMatchObject({ libraryVersionId: old.id, automatic: true });
@@ -503,8 +510,8 @@ describe('automatic host preparation', () => {
async (automatic) => {
const old = await publish();
const image = 'ghcr.io/example/plutainer:queued-version';
const id = enroll({ runtimeImages: [runtime, image] });
const other = enroll({ runtimeImages: [runtime] });
const id = enroll({ preparedRuntimeImages: [runtime, image] });
const other = enroll({ preparedRuntimeImages: [runtime] });
const { queueNativeCreation } =
await import('../../src/lib/server/provisioning/native.js');
for (let index = 0; index < 2; index++) {
@@ -521,7 +528,7 @@ describe('automatic host preparation', () => {
).toBe(true);
}
// Also cover pre-existing intents whose image is no longer cached.
heartbeat(id, { runtimeImages: [runtime] });
heartbeat(id, { preparedRuntimeImages: [runtime] });
const latest = await publish();
const status = preparation.hostPreparationStatus(hosts.getHost(id)!);
expect(status.items.filter((item) => item.label === image)).toHaveLength(0);
@@ -555,7 +562,7 @@ describe('automatic host preparation', () => {
vi.setSystemTime(Date.now() + 1);
hosts.reportHostOperation(id, download.id, { state: 'completed' });
heartbeat(id, {
runtimeImages: [runtime, image],
preparedRuntimeImages: [runtime, image],
libraries: [{ id: old.id, state: 'ready' }]
});
preparation.reconcileHostPreparation(id);
@@ -568,7 +575,7 @@ describe('automatic host preparation', () => {
it('retains an undelivered library download for an accepted intent until that intent is cancelled', async () => {
const old = await publish();
const id = enroll({ runtimeImages: [runtime] });
const id = enroll({ preparedRuntimeImages: [runtime] });
preparation.reconcileHostPreparation(id);
const download = operations(id)[0];
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
@@ -604,7 +611,7 @@ describe('automatic host preparation', () => {
it('finishes a delivered download before following a newer upload under a new operation ID', async () => {
const old = await publish();
const id = enroll({ runtimeImages: [runtime] });
const id = enroll({ preparedRuntimeImages: [runtime] });
preparation.reconcileHostPreparation(id);
const [first] = hosts.pendingHostOperations(id);
const latest = await publish();
@@ -613,7 +620,10 @@ describe('automatic host preparation', () => {
expect(hosts.getHostOperation(first.id)?.state).toBe('pending');
verified(id, old);
hosts.reportHostOperation(id, first.id, { state: 'completed' });
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
heartbeat(id, {
preparedRuntimeImages: [runtime],
libraries: [{ id: old.id, state: 'ready' }]
});
preparation.reconcileHostPreparation(id);
expect(operations(id)[0]).toMatchObject({
state: 'pending',
@@ -688,23 +698,26 @@ describe('automatic host preparation', () => {
it('blocks a library that cannot fit while preserving the host free-space reserve', async () => {
const version = await publish();
const id = enroll({
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
storageAvailable: 1_000 + version.sizeBytes - 1
});
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
expect(operations(id)).toEqual([]);
heartbeat(id, { runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes });
heartbeat(id, {
preparedRuntimeImages: [runtime],
storageAvailable: 1_000 + version.sizeBytes
});
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
});
it('returns an active download even when its in-progress bytes have consumed free space', async () => {
const version = await publish();
const id = enroll({ runtimeImages: [runtime] });
const id = enroll({ preparedRuntimeImages: [runtime] });
const operation = libraries.queueLibraryDownload(id, version.id);
heartbeat(id, {
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
storageAvailable: hosts.getHost(id)!.minimumFreeBytes
});
expect(libraries.queueLibraryDownload(id, version.id).id).toBe(operation.id);
@@ -715,7 +728,7 @@ describe('automatic host preparation', () => {
it('prepares T6 when the latest T5 upload cannot fit', async () => {
const oversized = await publish();
const small = await publish('t6', 'small');
const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_010 });
const id = enroll({ preparedRuntimeImages: [runtime], storageAvailable: 1_010 });
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
const status = preparation.reconcileHostPreparation(id);
expect(status.items).toEqual(
@@ -731,7 +744,7 @@ describe('automatic host preparation', () => {
it('does not count partial bytes from a different manifest toward download storage', async () => {
const version = await publish();
const id = enroll({ runtimeImages: [runtime] });
const id = enroll({ preparedRuntimeImages: [runtime] });
libraries.updateHostLibraryCaches(id, [
{
libraryVersionId: version.id,
@@ -741,7 +754,7 @@ describe('automatic host preparation', () => {
}
]);
heartbeat(id, {
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20
});
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
@@ -753,7 +766,7 @@ describe('automatic host preparation', () => {
it('resumes a failed verified-manifest download using only its remaining storage', async () => {
const version = await publish();
const id = enroll({ runtimeImages: [runtime] });
const id = enroll({ preparedRuntimeImages: [runtime] });
preparation.reconcileHostPreparation(id);
const operation = operations(id)[0];
libraries.updateHostLibraryCaches(id, [
@@ -768,11 +781,11 @@ describe('automatic host preparation', () => {
hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Connection reset' });
const failedAt = Date.now();
const free = hosts.getHost(id)!.minimumFreeBytes + version.sizeBytes - 80;
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
heartbeat(id, { preparedRuntimeImages: [runtime], storageAvailable: free });
preparation.reconcileHostPreparation(id);
expect(operations(id)).toHaveLength(1);
vi.setSystemTime(failedAt + 60_000);
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
heartbeat(id, { preparedRuntimeImages: [runtime], storageAvailable: free });
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
expect(operations(id)).toHaveLength(2);
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
@@ -800,7 +813,7 @@ describe('optional host Plutonium cache', () => {
(state) => {
const ready = {
...capable,
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
libraries: [{ id: 'game-files', state: 'ready' as const }]
};
const id = enroll(ready);
@@ -819,7 +832,7 @@ describe('optional host Plutonium cache', () => {
it('delivers newer deployments before queued cache work and before a cache replay', () => {
const id = enroll({
...capable,
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
libraries: [{ id: 'game-files', state: 'ready' }]
});
preparation.reconcileHostPreparation(id);
@@ -842,7 +855,7 @@ describe('optional host Plutonium cache', () => {
const ready = {
...inventory,
...capable,
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
libraries: [{ id: 'game-files', state: 'ready' as const }]
};
const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, ready);
@@ -903,7 +916,7 @@ describe('optional host Plutonium cache', () => {
const version = await publish();
const ready = {
...capable,
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
libraries: [{ id: version.id, state: 'ready' as const }],
plutoniumCache: { revision: 5354, bytes: 200 }
};
@@ -948,7 +961,7 @@ describe('optional host Plutonium cache', () => {
const pull = operations(id)[0];
expect(pull.kind).toBe('runtime.pull');
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
heartbeat(id, { ...capable, runtimeImages: [runtime] });
heartbeat(id, { ...capable, preparedRuntimeImages: [runtime] });
preparation.reconcileHostPreparation(id);
const download = operations(id)[0];
expect(download.kind).toBe('library.download');
@@ -956,7 +969,7 @@ describe('optional host Plutonium cache', () => {
hosts.reportHostOperation(id, download.id, { state: 'completed' });
const ready = {
...capable,
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
libraries: [{ id: version.id, state: 'ready' as const }]
};
heartbeat(id, ready);
@@ -1005,7 +1018,7 @@ describe('optional host Plutonium cache', () => {
it('refreshes for newer production releases and reconciles an interrupted refresh without duplicates', async () => {
const ready = {
...capable,
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
plutoniumCache: { revision: 5354, bytes: 25_000 }
};
const id = enroll(ready);
@@ -1048,7 +1061,7 @@ describe('optional host Plutonium cache', () => {
checkedAt: new Date(Date.now() - 60 * 60_000).toISOString()
});
if (evidence === 'failed') await release(5354, { error: 'Manifest unavailable' });
const id = enroll({ ...capable, runtimeImages: [runtime] });
const id = enroll({ ...capable, preparedRuntimeImages: [runtime] });
preparation.reconcileHostPreparation(id);
expect(operations(id)[0]).toMatchObject({
kind: 'plutonium.prepare',
@@ -1060,7 +1073,7 @@ describe('optional host Plutonium cache', () => {
).toBe('Latest production revision');
const cached = enroll({
...capable,
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
plutoniumCache: { revision: 5353, bytes: 200 }
});
preparation.reconcileHostPreparation(cached);
@@ -1071,7 +1084,7 @@ describe('optional host Plutonium cache', () => {
it('leaves agents without the cache capability unchanged', async () => {
await release(5355);
const id = enroll({
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
plutoniumCache: { revision: 5354, bytes: 200 }
});
expect(
@@ -1084,7 +1097,7 @@ describe('optional host Plutonium cache', () => {
const version = await publish();
const ready = {
...capable,
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
libraries: [{ id: version.id, state: 'ready' as const }]
};
const id = enroll(ready);
@@ -1135,7 +1148,7 @@ describe('optional host Plutonium cache', () => {
const version = await publish();
const id = enroll({
...capable,
runtimeImages: [runtime],
preparedRuntimeImages: [runtime],
storageAvailable: 1_000 + version.sizeBytes - 1
});
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
+1 -1
View File
@@ -11,7 +11,7 @@ const inventory = {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100 * 1024 ** 3,
runtimeImages: ['plutainer:test'],
preparedRuntimeImages: ['plutainer:test'],
libraries: [{ id: 'test-library', state: 'ready' }]
};
beforeEach(async () => {
+1 -1
View File
@@ -193,7 +193,7 @@ describe('native creation intents before placement', () => {
memoryTotal: 32 * 1024 ** 3,
memoryAvailable: 30 * 1024 ** 3,
storageAvailable: 100_000_000_000,
runtimeImages: [image]
preparedRuntimeImages: [image]
};
hosts.enrollHost(hosts.createEnrollment(ready.id).token, inventory);
const libraries = await import('../../src/lib/server/game-files/index.js');
+19 -4
View File
@@ -12,7 +12,7 @@ const inventory = {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100 * 1024 ** 3,
runtimeImages: ['plutainer:test'],
preparedRuntimeImages: ['plutainer:test'],
libraries: [{ id: 'test-library', state: 'ready' }]
};
beforeEach(async () => {
@@ -494,6 +494,21 @@ describe('native host enrollment and scoped authentication', () => {
});
describe('native host readiness and reservations', () => {
it.each([
{ preparedRuntimeImages: 'plutainer:test' },
{ preparedRuntimeImages: [123] },
{ preparedRuntimeImages: null }
])(
'rejects malformed prepared runtime inventory: $preparedRuntimeImages',
async ({ preparedRuntimeImages }) => {
const api = await enrolled();
expect(() =>
api.heartbeat(api.host.id, { ...inventory, preparedRuntimeImages })
).toThrow(/runtime image inventory/i);
expect(api.requireReadyHost(api.host.id).readiness).toBe('ready');
}
);
it('separates a connected agent from ready Docker, storage, runtime and library state', async () => {
const api = await enrolled();
expect(api.requireReadyHost(api.host.id).readiness).toBe('ready');
@@ -518,8 +533,8 @@ describe('native host readiness and reservations', () => {
});
it.each([
{ missing: { libraries: [] }, expected: 'needs-game-files' },
{ missing: { runtimeImages: [] }, expected: 'needs-runtime' },
{ missing: { runtimeImages: [], libraries: [] }, expected: 'needs-runtime' }
{ missing: { preparedRuntimeImages: [] }, expected: 'needs-runtime' },
{ missing: { preparedRuntimeImages: [], libraries: [] }, expected: 'needs-runtime' }
])(
'names missing content as $expected until a prepared heartbeat arrives',
async ({ missing, expected }) => {
@@ -545,7 +560,7 @@ describe('native host readiness and reservations', () => {
download.id,
runtime.id
]);
api.heartbeat(api.host.id, { ...inventory, runtimeImages: [], libraries: [] });
api.heartbeat(api.host.id, { ...inventory, preparedRuntimeImages: [], libraries: [] });
expect(api.pendingHostOperations(api.host.id).map((operation) => operation.id)).toEqual([
download.id,
runtime.id
+1 -1
View File
@@ -39,7 +39,7 @@ beforeEach(async () => {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 1_000_000_000_000,
runtimeImages: [image]
preparedRuntimeImages: [image]
};
enrollHost(createEnrollment(hostId).token, inventory);
const { createLibraryImport, updateHostLibraryCaches } =
+1 -1
View File
@@ -43,7 +43,7 @@ const inventory = () => ({
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100_000_000_000,
runtimeImages: [image],
preparedRuntimeImages: [image],
libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : []
});
beforeEach(async () => {
+1 -1
View File
@@ -34,7 +34,7 @@ beforeEach(async () => {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100_000_000_000,
runtimeImages: ['plutainer:test'],
preparedRuntimeImages: ['plutainer:test'],
libraries: [{ id: 'published-game-version', state: 'ready' }],
capabilities: ['instance.safety-override']
});
+2 -2
View File
@@ -47,7 +47,7 @@ async function heartbeat(hostId: string, cpuUsagePercent: number) {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100 * gib,
runtimeImages: [image],
preparedRuntimeImages: [image],
libraries: [{ id: libraryId, state: 'ready' }],
instanceResources: instances.map((instance) => ({
instanceId: instance.id,
@@ -92,7 +92,7 @@ beforeEach(async () => {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100 * gib,
runtimeImages: [image],
preparedRuntimeImages: [image],
libraries: [{ id: libraryId, state: 'ready' }]
});
const libraries = await import('../../src/lib/server/game-files/index.js');
+1 -1
View File
@@ -548,7 +548,7 @@ async function recoveryLaunch(action: 'create' | 'rebuild' | 'migration') {
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100_000_000_000,
runtimeImages: [config.defaultRuntimeImage],
preparedRuntimeImages: [config.defaultRuntimeImage],
libraries: [{ id: 'lib', state: 'ready' }]
});
const generation = action === 'create' ? 1 : 2;
+107 -16
View File
@@ -23,16 +23,31 @@ afterEach(async () => {
await fixture.cleanup();
});
async function cache(images: string[]) {
async function reportImages(
preparedImages: string[] | undefined,
cachedImages = preparedImages ?? []
) {
const hosts = await import('../../src/lib/server/hosts/index.js');
const host = hosts.getHost(prepared.host.id)!;
hosts.heartbeat(host.id, { ...host.inventory, runtimeImages: images });
hosts.heartbeat(host.id, {
...host.inventory,
runtimeImages: cachedImages,
preparedRuntimeImages: preparedImages
});
}
describe('runtime image launch policy', () => {
it.each([false, true])(
'rejects an uncached custom image at POST /api/servers (automatic: %s)',
async (automatic) => {
it.each([
{ automatic: false, cached: false, legacy: false },
{ automatic: true, cached: false, legacy: false },
{ automatic: false, cached: true, legacy: false },
{ automatic: true, cached: true, legacy: false },
{ automatic: false, cached: true, legacy: true },
{ automatic: true, cached: true, legacy: true }
])(
'rejects an unprepared custom image at POST /api/servers ($automatic, cached: $cached, legacy: $legacy)',
async ({ automatic, cached, legacy }) => {
await reportImages(legacy ? undefined : [], cached ? [custom] : []);
const { POST } = await import('../../src/routes/api/servers/+server.js');
const response = await POST({
request: new Request('https://controller.test/api/servers', {
@@ -57,13 +72,88 @@ describe('runtime image launch policy', () => {
);
it('allows the configured default before it is cached', async () => {
await cache([]);
await reportImages([]);
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
expect(queueNativeCreation(launch).ok).toBe(true);
});
it('keeps the cached configured default eligible with a legacy agent', async () => {
const { config } = await import('../../src/lib/server/config.js');
await reportImages(undefined, [config.defaultRuntimeImage, custom]);
const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
expect(nativeSelection({}, 't5').hostId).toBe(prepared.host.id);
const { getHost } = await import('../../src/lib/server/hosts/index.js');
expect(getHost(prepared.host.id)!.inventory.preparedRuntimeImages).toEqual([
config.defaultRuntimeImage
]);
});
it('ignores malformed preparation fields already stored by an older controller', async () => {
const { config } = await import('../../src/lib/server/config.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { getHost } = await import('../../src/lib/server/hosts/index.js');
const inventory = getHost(prepared.host.id)!.inventory;
for (const reported of [{}, 42, custom]) {
db()
.prepare('UPDATE ec_hosts SET inventory=? WHERE id=?')
.run(
JSON.stringify({
...inventory,
runtimeImages: [config.defaultRuntimeImage, custom],
preparedRuntimeImages: reported
}),
prepared.host.id
);
expect(getHost(prepared.host.id)).toMatchObject({
readiness: 'ready',
inventory: { preparedRuntimeImages: [config.defaultRuntimeImage] }
});
}
});
it('hides unrelated cached images from host list, detail, and page responses', async () => {
const { config } = await import('../../src/lib/server/config.js');
await reportImages(undefined, [config.defaultRuntimeImage, custom]);
const list = await import('../../src/routes/api/hosts/+server.js');
const detail = await import('../../src/routes/api/hosts/[id]/+server.js');
const { load } = await import('../../src/routes/hosts/+page.server.js');
const request = new Request('https://controller.test/api/hosts');
const bodies = [
await list.GET({ request }).json(),
await detail.GET({ request, params: { id: prepared.host.id } }).json(),
load()
];
for (const body of bodies) {
const host = 'hosts' in body ? body.hosts[0] : body.host;
expect(host.inventory.runtimeImages).toBeUndefined();
expect(host.inventory.preparedRuntimeImages).toEqual([config.defaultRuntimeImage]);
expect(JSON.stringify(body)).not.toContain(custom);
}
});
it('does not place an accepted runtime on another host that only caches it', async () => {
await reportImages([custom]);
const other = await prepareNativeHost();
const hosts = await import('../../src/lib/server/hosts/index.js');
hosts.heartbeat(other.host.id, {
...hosts.getHost(other.host.id)!.inventory,
runtimeImages: [custom]
});
expect(() => hosts.requireReadyHost(other.host.id, { image: custom })).toThrow(
/runtime image is not prepared/
);
const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js');
expect(() =>
choosePlacement({
hostId: other.host.id,
libraryId: other.versions.get('t5')!,
image: custom
})
).toThrow(/runtime image is not prepared/);
});
it('accepts prepared images only on hosts that have them', async () => {
await cache([custom]);
await reportImages([custom]);
const hosts = await import('../../src/lib/server/hosts/index.js');
const other = hosts.createHost({ name: 'Unprepared host' });
const { nativeLaunchSelection, nativeSelection } =
@@ -81,7 +171,7 @@ describe('runtime image launch policy', () => {
);
});
it('requires exact image references and does not treat a queued admin pull as cached', async () => {
it('requires exact prepared references and does not treat a queued admin pull as preparation', async () => {
const hosts = await import('../../src/lib/server/hosts/index.js');
hosts.queueHostOperation(prepared.host.id, 'runtime.pull', { image: custom });
const { nativeLaunchSelection } =
@@ -89,25 +179,25 @@ describe('runtime image launch policy', () => {
expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(
/prepared by an admin/
);
await cache([custom + '-other']);
await reportImages([custom + '-other']);
expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(
/prepared by an admin/
);
});
it('rechecks cached images when submitting a saved single-server preview', async () => {
await cache([custom]);
it('rechecks preparation when submitting a saved preview even if the image is still cached', async () => {
await reportImages([custom]);
const { previewNativeCreation, queueNativeCreation } =
await import('../../src/lib/server/provisioning/native.js');
const preview = previewNativeCreation({ ...launch, image: custom });
await cache([]);
await reportImages([], [custom]);
expect(queueNativeCreation({ ...launch, previewId: preview.id })).toMatchObject({
ok: false,
message: expect.stringContaining('prepared by an admin')
});
});
it('rejects uncached collection images and rechecks saved previews before writing any jobs', async () => {
it('rejects unprepared collection images and rechecks saved previews before writing any jobs', async () => {
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection, queueCollection } =
await import('../../src/lib/server/fleet/collections.js');
@@ -115,15 +205,16 @@ describe('runtime image launch policy', () => {
name: 'Image policy',
entries: [{ game: 't5', mapSlug: 'theater', count: 2 }]
});
await reportImages([], [custom]);
expect(() => previewCollection(blueprint.id, 'fill', { image: custom })).toThrow(
/prepared by an admin/
);
await cache([custom]);
await reportImages([custom]);
const preview = previewCollection(blueprint.id, 'fill', { image: custom });
const keys = Object.fromEntries(
preview.servers.map((server) => [server.name!, 'key-' + server.name])
);
await cache([]);
await reportImages([], [custom]);
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(
/prepared by an admin/
);
@@ -136,7 +227,7 @@ describe('runtime image launch policy', () => {
]) {
expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
}
await cache([custom]);
await reportImages([custom]);
expect(queueCollection({ previewId: preview.id, keys }).ok).toBe(true);
});
});