Cache Plutonium runtime files on hosts
This commit is contained in:
@@ -47,7 +47,7 @@ function heartbeat(id: string, overrides: HostInventory = {}) {
|
||||
}
|
||||
|
||||
function operations(id: string) {
|
||||
return hosts.listHostOperations(id).filter(operation => ['runtime.pull', 'library.download'].includes(operation.kind));
|
||||
return hosts.listHostOperations(id).filter(operation => ['runtime.pull', 'library.download', 'plutonium.prepare'].includes(operation.kind));
|
||||
}
|
||||
|
||||
async function publish(game: 't5' | 't6' = 't5', bytes = content) {
|
||||
@@ -85,10 +85,11 @@ describe('automatic host preparation', () => {
|
||||
} finally { cleanup.mockRestore(); }
|
||||
});
|
||||
|
||||
it('keeps re-enabled hosts disabled until a fresh runtime pull and both game libraries are verified', async () => {
|
||||
it.each([false, true])('keeps re-enabled hosts disabled until required files are verified, without waiting for the optional cache (supported: %s)', async supported => {
|
||||
const t5 = await publish('t5');
|
||||
const t6 = await publish('t6');
|
||||
const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })) };
|
||||
const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })),
|
||||
capabilities: [...inventory.capabilities!, ...(supported ? ['plutonium.cache'] : [])] };
|
||||
const id = enroll(readyInventory);
|
||||
verified(id, t5); verified(id, t6);
|
||||
const oldRuntime = hosts.queueHostOperation(id, 'runtime.pull', { image: runtime });
|
||||
@@ -126,6 +127,7 @@ describe('automatic host preparation', () => {
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(hosts.getHost(id)).toMatchObject({ mode: 'active', readiness: 'ready', enablingSince: null });
|
||||
expect(operations(id).filter(operation => operation.kind === 'library.download')).toHaveLength(2);
|
||||
expect(operations(id).filter(operation => operation.kind === 'plutonium.prepare')).toHaveLength(supported ? 1 : 0);
|
||||
});
|
||||
|
||||
it('requires both games when enabling even if one library is already cached', async () => {
|
||||
@@ -551,3 +553,247 @@ describe('automatic host preparation', () => {
|
||||
expect(libraries.getHostLibraryCache(id, version.id)?.bytesDownloaded).toBe(80);
|
||||
});
|
||||
});
|
||||
|
||||
describe('optional host Plutonium cache', () => {
|
||||
const capable = { capabilities: ['host.decommission', 'plutonium.cache'] };
|
||||
async function release(revision: number, overrides: { checkedAt?: string; error?: string | null } = {}) {
|
||||
const { writeSetting } = await import('../../src/lib/server/maintenance/settings.js');
|
||||
writeSetting('plutonium_release', { revision, checkedAt: new Date().toISOString(), error: null, ...overrides });
|
||||
}
|
||||
|
||||
it.each(['pending', 'accepted', 'running', 'unknown'] as const)('waits to warm while a deployment is %s', state => {
|
||||
const ready = { ...capable, runtimeImages: [runtime], libraries: [{ id: 'game-files', state: 'ready' as const }] };
|
||||
const id = enroll(ready);
|
||||
const deployment = hosts.queueHostOperation(id, 'instance.create', {});
|
||||
if (state !== 'pending') hosts.reportHostOperation(id, deployment.id, { state });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(true);
|
||||
expect(operations(id)).toEqual([]);
|
||||
hosts.reportHostOperation(id, deployment.id, { state: 'completed' });
|
||||
heartbeat(id, ready);
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toMatchObject([{ kind: 'plutonium.prepare' }]);
|
||||
});
|
||||
|
||||
it('delivers newer deployments before queued cache work and before a cache replay', () => {
|
||||
const id = enroll({ ...capable, runtimeImages: [runtime], libraries: [{ id: 'game-files', state: 'ready' }] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const cache = operations(id)[0];
|
||||
const deployment = hosts.queueHostOperation(id, 'instance.create', {});
|
||||
expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([deployment.id, cache.id]);
|
||||
hosts.reportHostOperation(id, cache.id, { state: 'running' });
|
||||
const change = hosts.queueHostOperation(id, 'files.stage', {});
|
||||
const delivery = hosts.pendingHostOperations(id, 'write').map(operation => operation.id);
|
||||
expect(delivery.at(-1)).toBe(cache.id);
|
||||
expect(delivery).toContain(deployment.id);
|
||||
expect(delivery).toContain(change.id);
|
||||
});
|
||||
|
||||
it('signals deferral on heartbeats for mutations while ignoring cache work and all observation lanes', async () => {
|
||||
const host = hosts.createHost({ name: 'Cache yielding host' });
|
||||
const ready = { ...inventory, ...capable, runtimeImages: [runtime], libraries: [{ id: 'game-files', state: 'ready' as const }] };
|
||||
const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, ready);
|
||||
preparation.reconcileHostPreparation(host.id);
|
||||
const cache = operations(host.id)[0];
|
||||
hosts.reportHostOperation(host.id, cache.id, { state: 'running' });
|
||||
for (const kind of ['host.inspect', 'instance.inspect', 'rcon.status-batch', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read']) {
|
||||
hosts.queueHostOperation(host.id, kind, {});
|
||||
}
|
||||
for (const command of ['status', 'getstatus', 'version']) hosts.queueHostOperation(host.id, 'rcon.execute', { command });
|
||||
// Work belonging to another host must not pause this one's cache.
|
||||
const other = enroll(ready);
|
||||
hosts.queueHostOperation(other, 'instance.restart', {});
|
||||
const { POST } = await import('../../src/routes/api/agent/heartbeat/+server.js');
|
||||
const poll = async () => {
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
const response = await POST({ request: new Request('https://controller.test/api/agent/heartbeat', {
|
||||
method: 'POST', headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ inventory: ready })
|
||||
}) });
|
||||
expect(response.status).toBe(200);
|
||||
return response.json();
|
||||
};
|
||||
expect(await poll()).toMatchObject({ deferPlutoniumPreparation: false });
|
||||
const change = hosts.queueHostOperation(host.id, 'rcon.execute', { command: 'map_restart' });
|
||||
expect(await poll()).toMatchObject({ deferPlutoniumPreparation: true });
|
||||
hosts.reportHostOperation(host.id, change.id, { state: 'failed', error: 'Command rejected' });
|
||||
expect(await poll()).toMatchObject({ deferPlutoniumPreparation: false });
|
||||
expect(operations(host.id).filter(operation => operation.kind === 'plutonium.prepare').map(operation => operation.id)).toEqual([cache.id]);
|
||||
});
|
||||
|
||||
it('lets newly published game files interrupt cache refresh, then resumes the same cache operation', async () => {
|
||||
const version = await publish();
|
||||
const ready = { ...capable, runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' as const }],
|
||||
plutoniumCache: { revision: 5354, bytes: 200 } };
|
||||
const id = enroll(ready);
|
||||
verified(id, version);
|
||||
await release(5355);
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const cache = operations(id)[0];
|
||||
expect(cache).toMatchObject({ kind: 'plutonium.prepare', payload: { revision: 5355 } });
|
||||
hosts.reportHostOperation(id, cache.id, { state: 'running' });
|
||||
const latest = await publish();
|
||||
heartbeat(id, ready);
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const download = operations(id)[0];
|
||||
expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } });
|
||||
expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(true);
|
||||
expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([download.id, cache.id]);
|
||||
verified(id, latest);
|
||||
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
||||
heartbeat(id, { ...ready, libraries: [{ id: latest.id, state: 'ready' }] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(false);
|
||||
expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([cache.id]);
|
||||
expect(operations(id).filter(operation => operation.kind === 'plutonium.prepare')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('warms only after required files, preserves launch readiness, and waits for fresh cache inventory', async () => {
|
||||
const version = await publish();
|
||||
await release(5354);
|
||||
const id = enroll(capable);
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const pull = operations(id)[0];
|
||||
expect(pull.kind).toBe('runtime.pull');
|
||||
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
|
||||
heartbeat(id, { ...capable, runtimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const download = operations(id)[0];
|
||||
expect(download.kind).toBe('library.download');
|
||||
verified(id, version);
|
||||
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
||||
const ready = { ...capable, runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' as const }] };
|
||||
heartbeat(id, ready);
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
||||
const cache = operations(id)[0];
|
||||
expect(cache).toMatchObject({ kind: 'plutonium.prepare', payload: {
|
||||
revision: 5354, minimumFreeBytes: hosts.getHost(id)!.minimumFreeBytes, automatic: true
|
||||
} });
|
||||
expect(hosts.requireReadyHost(id).readiness).toBe('ready');
|
||||
expect(hosts.pendingHostOperations(id, 'read')).toEqual([]);
|
||||
expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([cache.id]);
|
||||
expect(hosts.listHostOperationHistory(1, 'important', { host: id }).operations[0]).toMatchObject({
|
||||
id: cache.id, routine: false, targetPath: 'Production revision r5354'
|
||||
});
|
||||
preparation.reconcileHostPreparation(id);
|
||||
hosts.reportHostOperation(id, cache.id, { state: 'completed' });
|
||||
const pendingInventory = preparation.reconcileHostPreparation(id);
|
||||
expect(pendingInventory.items.find(item => item.kind === 'plutonium')).toMatchObject({ state: 'preparing' });
|
||||
expect(operations(id)).toHaveLength(3);
|
||||
heartbeat(id, { ...ready, plutoniumCache: { revision: 5354, bytes: 25_000 } });
|
||||
expect(preparation.reconcileHostPreparation(id).items.find(item => item.kind === 'plutonium')).toMatchObject({
|
||||
state: 'ready', detail: expect.stringContaining('r5354')
|
||||
});
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('refreshes for newer production releases and reconciles an interrupted refresh without duplicates', async () => {
|
||||
const ready = { ...capable, runtimeImages: [runtime], plutoniumCache: { revision: 5354, bytes: 25_000 } };
|
||||
const id = enroll(ready);
|
||||
await release(5354);
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toEqual([]);
|
||||
await release(5355);
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const cache = operations(id)[0];
|
||||
expect(cache.payload.revision).toBe(5355);
|
||||
hosts.reportHostOperation(id, cache.id, { state: 'unknown' });
|
||||
await release(5356);
|
||||
heartbeat(id, ready);
|
||||
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
closeDb();
|
||||
expect(preparation.reconcileHostPreparation(id).items.find(item => item.kind === 'plutonium')).toMatchObject({
|
||||
state: 'preparing', detail: expect.stringContaining('reconcile interrupted')
|
||||
});
|
||||
expect(operations(id)).toHaveLength(1);
|
||||
expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([cache.id]);
|
||||
hosts.reportHostOperation(id, cache.id, { state: 'completed' });
|
||||
heartbeat(id, { ...ready, plutoniumCache: { revision: 5356, bytes: 26_000 } });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(1);
|
||||
await release(5355);
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each(['missing', 'stale', 'failed'] as const)('uses the agent production manifest when release evidence is %s', async evidence => {
|
||||
if (evidence === 'stale') await release(5354, { checkedAt: new Date(Date.now() - 60 * 60_000).toISOString() });
|
||||
if (evidence === 'failed') await release(5354, { error: 'Manifest unavailable' });
|
||||
const id = enroll({ ...capable, runtimeImages: [runtime] });
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)[0]).toMatchObject({ kind: 'plutonium.prepare', payload: { revision: 0 } });
|
||||
expect(hosts.listHostOperationHistory(1, 'important', { host: id }).operations[0].targetPath).toBe('Latest production revision');
|
||||
const cached = enroll({ ...capable, runtimeImages: [runtime], plutoniumCache: { revision: 5353, bytes: 200 } });
|
||||
preparation.reconcileHostPreparation(cached);
|
||||
expect(operations(cached)).toEqual([]);
|
||||
});
|
||||
|
||||
it('leaves agents without the cache capability unchanged', async () => {
|
||||
await release(5355);
|
||||
const id = enroll({ runtimeImages: [runtime], plutoniumCache: { revision: 5354, bytes: 200 } });
|
||||
expect(preparation.reconcileHostPreparation(id).items.some(item => item.kind === 'plutonium')).toBe(false);
|
||||
expect(operations(id)).toEqual([]);
|
||||
});
|
||||
|
||||
it('backs off failed cache work without blocking launch readiness or newly published game files', async () => {
|
||||
const version = await publish();
|
||||
const ready = { ...capable, runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' as const }] };
|
||||
const id = enroll(ready);
|
||||
verified(id, version);
|
||||
await release(5354);
|
||||
preparation.reconcileHostPreparation(id);
|
||||
const cache = operations(id)[0];
|
||||
hosts.reportHostOperation(id, cache.id, { state: 'failed', error: 'Plutonium CDN unavailable' });
|
||||
const failedAt = Date.now();
|
||||
heartbeat(id, ready);
|
||||
const status = preparation.reconcileHostPreparation(id);
|
||||
expect(status).toMatchObject({ state: 'ready' });
|
||||
expect(status.items.find(item => item.kind === 'plutonium')).toMatchObject({
|
||||
state: 'blocked', detail: expect.stringContaining('Automatic retry')
|
||||
});
|
||||
expect(hosts.requireReadyHost(id).readiness).toBe('ready');
|
||||
expect(operations(id)).toHaveLength(1);
|
||||
// A changed or expired release must not bypass a failed cache's backoff.
|
||||
await release(5355);
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(1);
|
||||
const latest = await publish();
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } });
|
||||
verified(id, latest);
|
||||
hosts.reportHostOperation(id, operations(id)[0].id, { state: 'completed' });
|
||||
heartbeat(id, ready);
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(2);
|
||||
vi.setSystemTime(failedAt + 60_000);
|
||||
heartbeat(id, ready);
|
||||
preparation.reconcileHostPreparation(id);
|
||||
expect(operations(id)).toHaveLength(3);
|
||||
expect(operations(id)[0]).toMatchObject({ kind: 'plutonium.prepare', payload: { revision: 5355 } });
|
||||
});
|
||||
|
||||
it('waits for required library capacity before consuming space for the optional cache', async () => {
|
||||
const version = await publish();
|
||||
const id = enroll({ ...capable, runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes - 1 });
|
||||
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
||||
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
||||
expect(operations(id)).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([null, [], {}, { revision: 0, bytes: 1 }, { revision: 1.5, bytes: 1 }, { revision: 1, bytes: -1 }, { revision: 1, bytes: 0.5 }])(
|
||||
'rejects invalid cache inventory %j', cache => {
|
||||
const id = enroll();
|
||||
expect(() => heartbeat(id, { plutoniumCache: cache as HostInventory['plutoniumCache'] })).toThrow('Invalid Plutonium cache inventory');
|
||||
}
|
||||
);
|
||||
|
||||
it('retains only revision and byte count from cache inventory and clears an omitted cache', () => {
|
||||
const id = enroll({ plutoniumCache: { revision: 5354, bytes: 0, localPath: '/private/cache' } as HostInventory['plutoniumCache'] });
|
||||
expect(hosts.getHost(id)?.inventory.plutoniumCache).toEqual({ revision: 5354, bytes: 0 });
|
||||
heartbeat(id);
|
||||
expect(hosts.getHost(id)?.inventory.plutoniumCache).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user