This commit is contained in:
@@ -10,7 +10,7 @@ const inventory = {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: ['plutainer:test'],
|
||||
preparedRuntimeImages: ['plutainer:test'],
|
||||
libraries: [{ id: 'test-library', state: 'ready' }],
|
||||
capabilities: ['instance.cancel-create', 'operations.health-lane']
|
||||
};
|
||||
|
||||
@@ -18,7 +18,7 @@ const inventory = {
|
||||
cpuUsagePercent: 0,
|
||||
memoryTotal: 64 * GiB,
|
||||
memoryAvailable: 60 * GiB,
|
||||
runtimeImages: [image],
|
||||
preparedRuntimeImages: [image],
|
||||
libraries: [
|
||||
{ id: 't5-version', state: 'ready' },
|
||||
{ id: 't6-version', state: 'ready' }
|
||||
|
||||
@@ -35,7 +35,7 @@ beforeEach(async () => {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 1_000_000_000_000,
|
||||
runtimeImages: [image]
|
||||
preparedRuntimeImages: [image]
|
||||
};
|
||||
enrollHost(createEnrollment(hostId).token, inventory);
|
||||
const { createLibraryImport, updateHostLibraryCaches } =
|
||||
|
||||
@@ -47,7 +47,7 @@ beforeEach(async () => {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 1_000_000_000_000,
|
||||
runtimeImages: [image]
|
||||
preparedRuntimeImages: [image]
|
||||
};
|
||||
agentCredential = enrollHost(createEnrollment(hostId).token, inventory).credential;
|
||||
const { createLibraryImport, updateHostLibraryCaches } =
|
||||
|
||||
@@ -33,7 +33,7 @@ beforeEach(async () => {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: [runtime]
|
||||
preparedRuntimeImages: [runtime]
|
||||
});
|
||||
version = await publish();
|
||||
});
|
||||
|
||||
@@ -8,7 +8,7 @@ const inventory = {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: ['image'],
|
||||
preparedRuntimeImages: ['image'],
|
||||
libraries: [{ id: 'library', state: 'ready' }]
|
||||
};
|
||||
const lane: HostAgentLane = {
|
||||
|
||||
@@ -15,7 +15,7 @@ const inventory = {
|
||||
cpuUsagePercent: 0,
|
||||
memoryTotal: 64 * GiB,
|
||||
memoryAvailable: 60 * GiB,
|
||||
runtimeImages: ['image'],
|
||||
preparedRuntimeImages: ['image'],
|
||||
libraries: [
|
||||
{ id: 't5-library', state: 'ready' },
|
||||
{ id: 't6-library', state: 'ready' }
|
||||
@@ -46,6 +46,32 @@ afterEach(async () => {
|
||||
});
|
||||
|
||||
describe('cached launch availability', () => {
|
||||
it.each([{ preparedRuntimeImages: [] }, { preparedRuntimeImages: ['another:image'] }])(
|
||||
'excludes a cached image until it is prepared ($preparedRuntimeImages)',
|
||||
async ({ preparedRuntimeImages }) => {
|
||||
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
|
||||
const { launchAvailability } =
|
||||
await import('../../src/lib/server/hosts/availability.js');
|
||||
heartbeat(hostId, {
|
||||
...inventory,
|
||||
runtimeImages: ['image'],
|
||||
preparedRuntimeImages
|
||||
});
|
||||
const unavailable = launchAvailability([t5]);
|
||||
expect(unavailable.availability).toMatchObject({
|
||||
estimatedAvailable: 0,
|
||||
portsAvailable: 0
|
||||
});
|
||||
expect(unavailable.warnings.join(' ')).toContain('Waiting for an eligible host');
|
||||
|
||||
heartbeat(hostId, { ...inventory, runtimeImages: ['image'] });
|
||||
expect(launchAvailability([t5]).availability).toMatchObject({
|
||||
estimatedAvailable: 1,
|
||||
portsAvailable: 3
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
it('reads cached estimates without placement, historical scans, or repeated database reads', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const capacity = await import('../../src/lib/server/hosts/capacity.js');
|
||||
|
||||
@@ -14,7 +14,7 @@ const inventory = {
|
||||
protocolVersion: 1,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * GiB,
|
||||
runtimeImages: ['image'],
|
||||
preparedRuntimeImages: ['image'],
|
||||
libraries: [{ id: 'library', state: 'ready' }]
|
||||
};
|
||||
const sample = {
|
||||
|
||||
@@ -9,7 +9,7 @@ const inventory = {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: ['plutainer:test'],
|
||||
preparedRuntimeImages: ['plutainer:test'],
|
||||
libraries: [{ id: 'library', state: 'ready' }]
|
||||
};
|
||||
let fixture: Fixture;
|
||||
|
||||
@@ -18,7 +18,7 @@ const inventory: HostInventory = {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: [],
|
||||
preparedRuntimeImages: [],
|
||||
libraries: [],
|
||||
capabilities: ['host.decommission']
|
||||
};
|
||||
@@ -123,7 +123,7 @@ describe('automatic host preparation', () => {
|
||||
const t5 = await publish('t5');
|
||||
const t6 = await publish('t6');
|
||||
const readyInventory = {
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const })),
|
||||
capabilities: [
|
||||
...inventory.capabilities!,
|
||||
@@ -195,7 +195,7 @@ describe('automatic host preparation', () => {
|
||||
it('requires both games when enabling even if one library is already cached', async () => {
|
||||
const version = await publish('t5');
|
||||
const id = enroll({
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: version.id, state: 'ready' }]
|
||||
});
|
||||
verified(id, version);
|
||||
@@ -212,7 +212,7 @@ describe('automatic host preparation', () => {
|
||||
const t5 = await publish('t5');
|
||||
const t6 = await publish('t6');
|
||||
const readyInventory = {
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const }))
|
||||
};
|
||||
const id = enroll(readyInventory);
|
||||
@@ -310,7 +310,11 @@ describe('automatic host preparation', () => {
|
||||
expect(operations(host.id)).toHaveLength(1);
|
||||
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
const preparedInventory = { ...inventory, runtimeImages: [runtime], libraryCaches: [] };
|
||||
const preparedInventory = {
|
||||
...inventory,
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraryCaches: []
|
||||
};
|
||||
const runtimeHeartbeat = await reportHeartbeat({
|
||||
request: request('heartbeat', { inventory: preparedInventory })
|
||||
});
|
||||
@@ -369,7 +373,7 @@ describe('automatic host preparation', () => {
|
||||
|
||||
it('presents preparation without queuing work and prepares the configured runtime even when another image exists', async () => {
|
||||
const version = await publish();
|
||||
const id = enroll({ runtimeImages: ['another:runtime'] });
|
||||
const id = enroll({ preparedRuntimeImages: ['another:runtime'] });
|
||||
const host = hosts.getHost(id)!;
|
||||
const status = preparation.hostPreparationStatus(host);
|
||||
expect(status.items.map((item) => item.kind)).toEqual(['runtime', 'library']);
|
||||
@@ -406,7 +410,7 @@ describe('automatic host preparation', () => {
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(1);
|
||||
|
||||
heartbeat(id, { runtimeImages: [runtime] });
|
||||
heartbeat(id, { preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(2);
|
||||
const download = operations(id)[0];
|
||||
@@ -417,7 +421,7 @@ describe('automatic host preparation', () => {
|
||||
verified(id, version);
|
||||
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
||||
heartbeat(id, {
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: version.id, state: 'ready' }]
|
||||
});
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
||||
@@ -432,7 +436,7 @@ describe('automatic host preparation', () => {
|
||||
game: 't5',
|
||||
files: [{ path: 'zone/game.ff', size: content.length, lastModified: 0 }]
|
||||
});
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
const expected = new Map([latest, t6].map((version) => [version.id, version]));
|
||||
const downloaded: string[] = [];
|
||||
for (let index = 0; index < expected.size; index++) {
|
||||
@@ -452,7 +456,7 @@ describe('automatic host preparation', () => {
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(index + 1);
|
||||
heartbeat(id, {
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: downloaded.map((id) => ({ id, state: 'ready' }))
|
||||
});
|
||||
}
|
||||
@@ -462,12 +466,15 @@ describe('automatic host preparation', () => {
|
||||
});
|
||||
|
||||
it('picks up publication after enrollment and keeps verified historical versions intact', async () => {
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toEqual([]);
|
||||
const old = await publish();
|
||||
verified(id, old);
|
||||
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
|
||||
heartbeat(id, {
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: old.id, state: 'ready' }]
|
||||
});
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
||||
const latest = await publish();
|
||||
preparation.reconcileHostPreparation(id);
|
||||
@@ -479,7 +486,7 @@ describe('automatic host preparation', () => {
|
||||
|
||||
it('replaces an undelivered automatic download when newer files finish uploading', async () => {
|
||||
const old = await publish();
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const first = operations(id)[0];
|
||||
expect(first.payload).toMatchObject({ libraryVersionId: old.id, automatic: true });
|
||||
@@ -503,8 +510,8 @@ describe('automatic host preparation', () => {
|
||||
async (automatic) => {
|
||||
const old = await publish();
|
||||
const image = 'ghcr.io/example/plutainer:queued-version';
|
||||
const id = enroll({ runtimeImages: [runtime, image] });
|
||||
const other = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime, image] });
|
||||
const other = enroll({ preparedRuntimeImages: [runtime] });
|
||||
const { queueNativeCreation } =
|
||||
await import('../../src/lib/server/provisioning/native.js');
|
||||
for (let index = 0; index < 2; index++) {
|
||||
@@ -521,7 +528,7 @@ describe('automatic host preparation', () => {
|
||||
).toBe(true);
|
||||
}
|
||||
// Also cover pre-existing intents whose image is no longer cached.
|
||||
heartbeat(id, { runtimeImages: [runtime] });
|
||||
heartbeat(id, { preparedRuntimeImages: [runtime] });
|
||||
const latest = await publish();
|
||||
const status = preparation.hostPreparationStatus(hosts.getHost(id)!);
|
||||
expect(status.items.filter((item) => item.label === image)).toHaveLength(0);
|
||||
@@ -555,7 +562,7 @@ describe('automatic host preparation', () => {
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
||||
heartbeat(id, {
|
||||
runtimeImages: [runtime, image],
|
||||
preparedRuntimeImages: [runtime, image],
|
||||
libraries: [{ id: old.id, state: 'ready' }]
|
||||
});
|
||||
preparation.reconcileHostPreparation(id);
|
||||
@@ -568,7 +575,7 @@ describe('automatic host preparation', () => {
|
||||
|
||||
it('retains an undelivered library download for an accepted intent until that intent is cancelled', async () => {
|
||||
const old = await publish();
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const download = operations(id)[0];
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
@@ -604,7 +611,7 @@ describe('automatic host preparation', () => {
|
||||
|
||||
it('finishes a delivered download before following a newer upload under a new operation ID', async () => {
|
||||
const old = await publish();
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const [first] = hosts.pendingHostOperations(id);
|
||||
const latest = await publish();
|
||||
@@ -613,7 +620,10 @@ describe('automatic host preparation', () => {
|
||||
expect(hosts.getHostOperation(first.id)?.state).toBe('pending');
|
||||
verified(id, old);
|
||||
hosts.reportHostOperation(id, first.id, { state: 'completed' });
|
||||
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
|
||||
heartbeat(id, {
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: old.id, state: 'ready' }]
|
||||
});
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)[0]).toMatchObject({
|
||||
state: 'pending',
|
||||
@@ -688,23 +698,26 @@ describe('automatic host preparation', () => {
|
||||
it('blocks a library that cannot fit while preserving the host free-space reserve', async () => {
|
||||
const version = await publish();
|
||||
const id = enroll({
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
storageAvailable: 1_000 + version.sizeBytes - 1
|
||||
});
|
||||
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
||||
expect(operations(id)).toEqual([]);
|
||||
heartbeat(id, { runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes });
|
||||
heartbeat(id, {
|
||||
preparedRuntimeImages: [runtime],
|
||||
storageAvailable: 1_000 + version.sizeBytes
|
||||
});
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
||||
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
||||
});
|
||||
|
||||
it('returns an active download even when its in-progress bytes have consumed free space', async () => {
|
||||
const version = await publish();
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
const operation = libraries.queueLibraryDownload(id, version.id);
|
||||
heartbeat(id, {
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
storageAvailable: hosts.getHost(id)!.minimumFreeBytes
|
||||
});
|
||||
expect(libraries.queueLibraryDownload(id, version.id).id).toBe(operation.id);
|
||||
@@ -715,7 +728,7 @@ describe('automatic host preparation', () => {
|
||||
it('prepares T6 when the latest T5 upload cannot fit', async () => {
|
||||
const oversized = await publish();
|
||||
const small = await publish('t6', 'small');
|
||||
const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_010 });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime], storageAvailable: 1_010 });
|
||||
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
||||
const status = preparation.reconcileHostPreparation(id);
|
||||
expect(status.items).toEqual(
|
||||
@@ -731,7 +744,7 @@ describe('automatic host preparation', () => {
|
||||
|
||||
it('does not count partial bytes from a different manifest toward download storage', async () => {
|
||||
const version = await publish();
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
libraries.updateHostLibraryCaches(id, [
|
||||
{
|
||||
libraryVersionId: version.id,
|
||||
@@ -741,7 +754,7 @@ describe('automatic host preparation', () => {
|
||||
}
|
||||
]);
|
||||
heartbeat(id, {
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20
|
||||
});
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
||||
@@ -753,7 +766,7 @@ describe('automatic host preparation', () => {
|
||||
|
||||
it('resumes a failed verified-manifest download using only its remaining storage', async () => {
|
||||
const version = await publish();
|
||||
const id = enroll({ runtimeImages: [runtime] });
|
||||
const id = enroll({ preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const operation = operations(id)[0];
|
||||
libraries.updateHostLibraryCaches(id, [
|
||||
@@ -768,11 +781,11 @@ describe('automatic host preparation', () => {
|
||||
hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Connection reset' });
|
||||
const failedAt = Date.now();
|
||||
const free = hosts.getHost(id)!.minimumFreeBytes + version.sizeBytes - 80;
|
||||
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
|
||||
heartbeat(id, { preparedRuntimeImages: [runtime], storageAvailable: free });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(1);
|
||||
vi.setSystemTime(failedAt + 60_000);
|
||||
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
|
||||
heartbeat(id, { preparedRuntimeImages: [runtime], storageAvailable: free });
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
||||
expect(operations(id)).toHaveLength(2);
|
||||
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
||||
@@ -800,7 +813,7 @@ describe('optional host Plutonium cache', () => {
|
||||
(state) => {
|
||||
const ready = {
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: 'game-files', state: 'ready' as const }]
|
||||
};
|
||||
const id = enroll(ready);
|
||||
@@ -819,7 +832,7 @@ describe('optional host Plutonium cache', () => {
|
||||
it('delivers newer deployments before queued cache work and before a cache replay', () => {
|
||||
const id = enroll({
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: 'game-files', state: 'ready' }]
|
||||
});
|
||||
preparation.reconcileHostPreparation(id);
|
||||
@@ -842,7 +855,7 @@ describe('optional host Plutonium cache', () => {
|
||||
const ready = {
|
||||
...inventory,
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: 'game-files', state: 'ready' as const }]
|
||||
};
|
||||
const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, ready);
|
||||
@@ -903,7 +916,7 @@ describe('optional host Plutonium cache', () => {
|
||||
const version = await publish();
|
||||
const ready = {
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: version.id, state: 'ready' as const }],
|
||||
plutoniumCache: { revision: 5354, bytes: 200 }
|
||||
};
|
||||
@@ -948,7 +961,7 @@ describe('optional host Plutonium cache', () => {
|
||||
const pull = operations(id)[0];
|
||||
expect(pull.kind).toBe('runtime.pull');
|
||||
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
|
||||
heartbeat(id, { ...capable, runtimeImages: [runtime] });
|
||||
heartbeat(id, { ...capable, preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const download = operations(id)[0];
|
||||
expect(download.kind).toBe('library.download');
|
||||
@@ -956,7 +969,7 @@ describe('optional host Plutonium cache', () => {
|
||||
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
||||
const ready = {
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: version.id, state: 'ready' as const }]
|
||||
};
|
||||
heartbeat(id, ready);
|
||||
@@ -1005,7 +1018,7 @@ describe('optional host Plutonium cache', () => {
|
||||
it('refreshes for newer production releases and reconciles an interrupted refresh without duplicates', async () => {
|
||||
const ready = {
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
plutoniumCache: { revision: 5354, bytes: 25_000 }
|
||||
};
|
||||
const id = enroll(ready);
|
||||
@@ -1048,7 +1061,7 @@ describe('optional host Plutonium cache', () => {
|
||||
checkedAt: new Date(Date.now() - 60 * 60_000).toISOString()
|
||||
});
|
||||
if (evidence === 'failed') await release(5354, { error: 'Manifest unavailable' });
|
||||
const id = enroll({ ...capable, runtimeImages: [runtime] });
|
||||
const id = enroll({ ...capable, preparedRuntimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)[0]).toMatchObject({
|
||||
kind: 'plutonium.prepare',
|
||||
@@ -1060,7 +1073,7 @@ describe('optional host Plutonium cache', () => {
|
||||
).toBe('Latest production revision');
|
||||
const cached = enroll({
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
plutoniumCache: { revision: 5353, bytes: 200 }
|
||||
});
|
||||
preparation.reconcileHostPreparation(cached);
|
||||
@@ -1071,7 +1084,7 @@ describe('optional host Plutonium cache', () => {
|
||||
it('leaves agents without the cache capability unchanged', async () => {
|
||||
await release(5355);
|
||||
const id = enroll({
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
plutoniumCache: { revision: 5354, bytes: 200 }
|
||||
});
|
||||
expect(
|
||||
@@ -1084,7 +1097,7 @@ describe('optional host Plutonium cache', () => {
|
||||
const version = await publish();
|
||||
const ready = {
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
libraries: [{ id: version.id, state: 'ready' as const }]
|
||||
};
|
||||
const id = enroll(ready);
|
||||
@@ -1135,7 +1148,7 @@ describe('optional host Plutonium cache', () => {
|
||||
const version = await publish();
|
||||
const id = enroll({
|
||||
...capable,
|
||||
runtimeImages: [runtime],
|
||||
preparedRuntimeImages: [runtime],
|
||||
storageAvailable: 1_000 + version.sizeBytes - 1
|
||||
});
|
||||
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
||||
|
||||
@@ -11,7 +11,7 @@ const inventory = {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: ['plutainer:test'],
|
||||
preparedRuntimeImages: ['plutainer:test'],
|
||||
libraries: [{ id: 'test-library', state: 'ready' }]
|
||||
};
|
||||
beforeEach(async () => {
|
||||
|
||||
@@ -193,7 +193,7 @@ describe('native creation intents before placement', () => {
|
||||
memoryTotal: 32 * 1024 ** 3,
|
||||
memoryAvailable: 30 * 1024 ** 3,
|
||||
storageAvailable: 100_000_000_000,
|
||||
runtimeImages: [image]
|
||||
preparedRuntimeImages: [image]
|
||||
};
|
||||
hosts.enrollHost(hosts.createEnrollment(ready.id).token, inventory);
|
||||
const libraries = await import('../../src/lib/server/game-files/index.js');
|
||||
|
||||
@@ -12,7 +12,7 @@ const inventory = {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * 1024 ** 3,
|
||||
runtimeImages: ['plutainer:test'],
|
||||
preparedRuntimeImages: ['plutainer:test'],
|
||||
libraries: [{ id: 'test-library', state: 'ready' }]
|
||||
};
|
||||
beforeEach(async () => {
|
||||
@@ -494,6 +494,21 @@ describe('native host enrollment and scoped authentication', () => {
|
||||
});
|
||||
|
||||
describe('native host readiness and reservations', () => {
|
||||
it.each([
|
||||
{ preparedRuntimeImages: 'plutainer:test' },
|
||||
{ preparedRuntimeImages: [123] },
|
||||
{ preparedRuntimeImages: null }
|
||||
])(
|
||||
'rejects malformed prepared runtime inventory: $preparedRuntimeImages',
|
||||
async ({ preparedRuntimeImages }) => {
|
||||
const api = await enrolled();
|
||||
expect(() =>
|
||||
api.heartbeat(api.host.id, { ...inventory, preparedRuntimeImages })
|
||||
).toThrow(/runtime image inventory/i);
|
||||
expect(api.requireReadyHost(api.host.id).readiness).toBe('ready');
|
||||
}
|
||||
);
|
||||
|
||||
it('separates a connected agent from ready Docker, storage, runtime and library state', async () => {
|
||||
const api = await enrolled();
|
||||
expect(api.requireReadyHost(api.host.id).readiness).toBe('ready');
|
||||
@@ -518,8 +533,8 @@ describe('native host readiness and reservations', () => {
|
||||
});
|
||||
it.each([
|
||||
{ missing: { libraries: [] }, expected: 'needs-game-files' },
|
||||
{ missing: { runtimeImages: [] }, expected: 'needs-runtime' },
|
||||
{ missing: { runtimeImages: [], libraries: [] }, expected: 'needs-runtime' }
|
||||
{ missing: { preparedRuntimeImages: [] }, expected: 'needs-runtime' },
|
||||
{ missing: { preparedRuntimeImages: [], libraries: [] }, expected: 'needs-runtime' }
|
||||
])(
|
||||
'names missing content as $expected until a prepared heartbeat arrives',
|
||||
async ({ missing, expected }) => {
|
||||
@@ -545,7 +560,7 @@ describe('native host readiness and reservations', () => {
|
||||
download.id,
|
||||
runtime.id
|
||||
]);
|
||||
api.heartbeat(api.host.id, { ...inventory, runtimeImages: [], libraries: [] });
|
||||
api.heartbeat(api.host.id, { ...inventory, preparedRuntimeImages: [], libraries: [] });
|
||||
expect(api.pendingHostOperations(api.host.id).map((operation) => operation.id)).toEqual([
|
||||
download.id,
|
||||
runtime.id
|
||||
|
||||
@@ -39,7 +39,7 @@ beforeEach(async () => {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 1_000_000_000_000,
|
||||
runtimeImages: [image]
|
||||
preparedRuntimeImages: [image]
|
||||
};
|
||||
enrollHost(createEnrollment(hostId).token, inventory);
|
||||
const { createLibraryImport, updateHostLibraryCaches } =
|
||||
|
||||
@@ -43,7 +43,7 @@ const inventory = () => ({
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100_000_000_000,
|
||||
runtimeImages: [image],
|
||||
preparedRuntimeImages: [image],
|
||||
libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : []
|
||||
});
|
||||
beforeEach(async () => {
|
||||
|
||||
@@ -34,7 +34,7 @@ beforeEach(async () => {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100_000_000_000,
|
||||
runtimeImages: ['plutainer:test'],
|
||||
preparedRuntimeImages: ['plutainer:test'],
|
||||
libraries: [{ id: 'published-game-version', state: 'ready' }],
|
||||
capabilities: ['instance.safety-override']
|
||||
});
|
||||
|
||||
@@ -47,7 +47,7 @@ async function heartbeat(hostId: string, cpuUsagePercent: number) {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * gib,
|
||||
runtimeImages: [image],
|
||||
preparedRuntimeImages: [image],
|
||||
libraries: [{ id: libraryId, state: 'ready' }],
|
||||
instanceResources: instances.map((instance) => ({
|
||||
instanceId: instance.id,
|
||||
@@ -92,7 +92,7 @@ beforeEach(async () => {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100 * gib,
|
||||
runtimeImages: [image],
|
||||
preparedRuntimeImages: [image],
|
||||
libraries: [{ id: libraryId, state: 'ready' }]
|
||||
});
|
||||
const libraries = await import('../../src/lib/server/game-files/index.js');
|
||||
|
||||
@@ -548,7 +548,7 @@ async function recoveryLaunch(action: 'create' | 'rebuild' | 'migration') {
|
||||
dockerRunning: true,
|
||||
directoriesWritable: true,
|
||||
storageAvailable: 100_000_000_000,
|
||||
runtimeImages: [config.defaultRuntimeImage],
|
||||
preparedRuntimeImages: [config.defaultRuntimeImage],
|
||||
libraries: [{ id: 'lib', state: 'ready' }]
|
||||
});
|
||||
const generation = action === 'create' ? 1 : 2;
|
||||
|
||||
@@ -23,16 +23,31 @@ afterEach(async () => {
|
||||
await fixture.cleanup();
|
||||
});
|
||||
|
||||
async function cache(images: string[]) {
|
||||
async function reportImages(
|
||||
preparedImages: string[] | undefined,
|
||||
cachedImages = preparedImages ?? []
|
||||
) {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const host = hosts.getHost(prepared.host.id)!;
|
||||
hosts.heartbeat(host.id, { ...host.inventory, runtimeImages: images });
|
||||
hosts.heartbeat(host.id, {
|
||||
...host.inventory,
|
||||
runtimeImages: cachedImages,
|
||||
preparedRuntimeImages: preparedImages
|
||||
});
|
||||
}
|
||||
|
||||
describe('runtime image launch policy', () => {
|
||||
it.each([false, true])(
|
||||
'rejects an uncached custom image at POST /api/servers (automatic: %s)',
|
||||
async (automatic) => {
|
||||
it.each([
|
||||
{ automatic: false, cached: false, legacy: false },
|
||||
{ automatic: true, cached: false, legacy: false },
|
||||
{ automatic: false, cached: true, legacy: false },
|
||||
{ automatic: true, cached: true, legacy: false },
|
||||
{ automatic: false, cached: true, legacy: true },
|
||||
{ automatic: true, cached: true, legacy: true }
|
||||
])(
|
||||
'rejects an unprepared custom image at POST /api/servers ($automatic, cached: $cached, legacy: $legacy)',
|
||||
async ({ automatic, cached, legacy }) => {
|
||||
await reportImages(legacy ? undefined : [], cached ? [custom] : []);
|
||||
const { POST } = await import('../../src/routes/api/servers/+server.js');
|
||||
const response = await POST({
|
||||
request: new Request('https://controller.test/api/servers', {
|
||||
@@ -57,13 +72,88 @@ describe('runtime image launch policy', () => {
|
||||
);
|
||||
|
||||
it('allows the configured default before it is cached', async () => {
|
||||
await cache([]);
|
||||
await reportImages([]);
|
||||
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
||||
expect(queueNativeCreation(launch).ok).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps the cached configured default eligible with a legacy agent', async () => {
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
await reportImages(undefined, [config.defaultRuntimeImage, custom]);
|
||||
const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
|
||||
expect(nativeSelection({}, 't5').hostId).toBe(prepared.host.id);
|
||||
const { getHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
expect(getHost(prepared.host.id)!.inventory.preparedRuntimeImages).toEqual([
|
||||
config.defaultRuntimeImage
|
||||
]);
|
||||
});
|
||||
|
||||
it('ignores malformed preparation fields already stored by an older controller', async () => {
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { getHost } = await import('../../src/lib/server/hosts/index.js');
|
||||
const inventory = getHost(prepared.host.id)!.inventory;
|
||||
for (const reported of [{}, 42, custom]) {
|
||||
db()
|
||||
.prepare('UPDATE ec_hosts SET inventory=? WHERE id=?')
|
||||
.run(
|
||||
JSON.stringify({
|
||||
...inventory,
|
||||
runtimeImages: [config.defaultRuntimeImage, custom],
|
||||
preparedRuntimeImages: reported
|
||||
}),
|
||||
prepared.host.id
|
||||
);
|
||||
expect(getHost(prepared.host.id)).toMatchObject({
|
||||
readiness: 'ready',
|
||||
inventory: { preparedRuntimeImages: [config.defaultRuntimeImage] }
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('hides unrelated cached images from host list, detail, and page responses', async () => {
|
||||
const { config } = await import('../../src/lib/server/config.js');
|
||||
await reportImages(undefined, [config.defaultRuntimeImage, custom]);
|
||||
const list = await import('../../src/routes/api/hosts/+server.js');
|
||||
const detail = await import('../../src/routes/api/hosts/[id]/+server.js');
|
||||
const { load } = await import('../../src/routes/hosts/+page.server.js');
|
||||
const request = new Request('https://controller.test/api/hosts');
|
||||
const bodies = [
|
||||
await list.GET({ request }).json(),
|
||||
await detail.GET({ request, params: { id: prepared.host.id } }).json(),
|
||||
load()
|
||||
];
|
||||
for (const body of bodies) {
|
||||
const host = 'hosts' in body ? body.hosts[0] : body.host;
|
||||
expect(host.inventory.runtimeImages).toBeUndefined();
|
||||
expect(host.inventory.preparedRuntimeImages).toEqual([config.defaultRuntimeImage]);
|
||||
expect(JSON.stringify(body)).not.toContain(custom);
|
||||
}
|
||||
});
|
||||
|
||||
it('does not place an accepted runtime on another host that only caches it', async () => {
|
||||
await reportImages([custom]);
|
||||
const other = await prepareNativeHost();
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
hosts.heartbeat(other.host.id, {
|
||||
...hosts.getHost(other.host.id)!.inventory,
|
||||
runtimeImages: [custom]
|
||||
});
|
||||
expect(() => hosts.requireReadyHost(other.host.id, { image: custom })).toThrow(
|
||||
/runtime image is not prepared/
|
||||
);
|
||||
const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js');
|
||||
expect(() =>
|
||||
choosePlacement({
|
||||
hostId: other.host.id,
|
||||
libraryId: other.versions.get('t5')!,
|
||||
image: custom
|
||||
})
|
||||
).toThrow(/runtime image is not prepared/);
|
||||
});
|
||||
|
||||
it('accepts prepared images only on hosts that have them', async () => {
|
||||
await cache([custom]);
|
||||
await reportImages([custom]);
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const other = hosts.createHost({ name: 'Unprepared host' });
|
||||
const { nativeLaunchSelection, nativeSelection } =
|
||||
@@ -81,7 +171,7 @@ describe('runtime image launch policy', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('requires exact image references and does not treat a queued admin pull as cached', async () => {
|
||||
it('requires exact prepared references and does not treat a queued admin pull as preparation', async () => {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
hosts.queueHostOperation(prepared.host.id, 'runtime.pull', { image: custom });
|
||||
const { nativeLaunchSelection } =
|
||||
@@ -89,25 +179,25 @@ describe('runtime image launch policy', () => {
|
||||
expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(
|
||||
/prepared by an admin/
|
||||
);
|
||||
await cache([custom + '-other']);
|
||||
await reportImages([custom + '-other']);
|
||||
expect(() => nativeLaunchSelection({ image: custom }, 't5')).toThrow(
|
||||
/prepared by an admin/
|
||||
);
|
||||
});
|
||||
|
||||
it('rechecks cached images when submitting a saved single-server preview', async () => {
|
||||
await cache([custom]);
|
||||
it('rechecks preparation when submitting a saved preview even if the image is still cached', async () => {
|
||||
await reportImages([custom]);
|
||||
const { previewNativeCreation, queueNativeCreation } =
|
||||
await import('../../src/lib/server/provisioning/native.js');
|
||||
const preview = previewNativeCreation({ ...launch, image: custom });
|
||||
await cache([]);
|
||||
await reportImages([], [custom]);
|
||||
expect(queueNativeCreation({ ...launch, previewId: preview.id })).toMatchObject({
|
||||
ok: false,
|
||||
message: expect.stringContaining('prepared by an admin')
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects uncached collection images and rechecks saved previews before writing any jobs', async () => {
|
||||
it('rejects unprepared collection images and rechecks saved previews before writing any jobs', async () => {
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { previewCollection, queueCollection } =
|
||||
await import('../../src/lib/server/fleet/collections.js');
|
||||
@@ -115,15 +205,16 @@ describe('runtime image launch policy', () => {
|
||||
name: 'Image policy',
|
||||
entries: [{ game: 't5', mapSlug: 'theater', count: 2 }]
|
||||
});
|
||||
await reportImages([], [custom]);
|
||||
expect(() => previewCollection(blueprint.id, 'fill', { image: custom })).toThrow(
|
||||
/prepared by an admin/
|
||||
);
|
||||
await cache([custom]);
|
||||
await reportImages([custom]);
|
||||
const preview = previewCollection(blueprint.id, 'fill', { image: custom });
|
||||
const keys = Object.fromEntries(
|
||||
preview.servers.map((server) => [server.name!, 'key-' + server.name])
|
||||
);
|
||||
await cache([]);
|
||||
await reportImages([], [custom]);
|
||||
expect(() => queueCollection({ previewId: preview.id, keys })).toThrow(
|
||||
/prepared by an admin/
|
||||
);
|
||||
@@ -136,7 +227,7 @@ describe('runtime image launch policy', () => {
|
||||
]) {
|
||||
expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
|
||||
}
|
||||
await cache([custom]);
|
||||
await reportImages([custom]);
|
||||
expect(queueCollection({ previewId: preview.id, keys }).ok).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user