800 lines
46 KiB
TypeScript
800 lines
46 KiB
TypeScript
import { createHash } from 'node:crypto';
|
|
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import type { GameLibraryVersion } from '../../src/lib/game-files.js';
|
|
import type { HostInventory } from '../../src/lib/hosts.js';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
let fixture: Fixture;
|
|
let hosts: typeof import('../../src/lib/server/hosts/index.js');
|
|
let libraries: typeof import('../../src/lib/server/game-files/index.js');
|
|
let preparation: typeof import('../../src/lib/server/hosts/preparation.js');
|
|
const runtime = 'ghcr.io/example/plutainer:provisioning-test';
|
|
const initialTime = Date.parse('2026-10-04T12:00:00Z');
|
|
const content = 'a'.repeat(100);
|
|
const inventory: HostInventory = {
|
|
hostname: 'game-host', protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
|
|
storageAvailable: 100 * 1024 ** 3, runtimeImages: [], libraries: [], capabilities: ['host.decommission']
|
|
};
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture([]);
|
|
vi.stubEnv('PLUTAINER_IMAGE', runtime);
|
|
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'game-files'));
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(initialTime);
|
|
hosts = await import('../../src/lib/server/hosts/index.js');
|
|
libraries = await import('../../src/lib/server/game-files/index.js');
|
|
preparation = await import('../../src/lib/server/hosts/preparation.js');
|
|
});
|
|
|
|
afterEach(async () => {
|
|
vi.useRealTimers();
|
|
vi.unstubAllEnvs();
|
|
await fixture.cleanup();
|
|
});
|
|
|
|
function enroll(overrides: HostInventory = {}) {
|
|
const host = hosts.createHost({ name: 'Automatic game host' });
|
|
hosts.enrollHost(hosts.createEnrollment(host.id).token, { ...inventory, ...overrides });
|
|
return host.id;
|
|
}
|
|
|
|
function heartbeat(id: string, overrides: HostInventory = {}) {
|
|
vi.setSystemTime(Date.now() + 1);
|
|
return hosts.heartbeat(id, { ...inventory, ...overrides });
|
|
}
|
|
|
|
function operations(id: string) {
|
|
return hosts.listHostOperations(id).filter(operation => ['runtime.pull', 'library.download', 'plutonium.prepare'].includes(operation.kind));
|
|
}
|
|
|
|
async function publish(game: 't5' | 't6' = 't5', bytes = content) {
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const upload = libraries.createLibraryUpload({
|
|
game, files: [{ path: 'zone/game.ff', size: bytes.length, lastModified: 0 }]
|
|
});
|
|
await libraries.uploadBrowserFile(upload.id, 'zone/game.ff', {
|
|
offset: 0, size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex')
|
|
}, new Response(bytes).body);
|
|
return libraries.publishBrowserUpload(upload.id);
|
|
}
|
|
|
|
function verified(id: string, version: GameLibraryVersion) {
|
|
libraries.updateHostLibraryCaches(id, [{
|
|
libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256
|
|
}]);
|
|
}
|
|
|
|
describe('automatic host preparation', () => {
|
|
it('accepts heartbeats without running controller library cleanup', async () => {
|
|
const host = hosts.createHost({ name: 'Heartbeat host' });
|
|
const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
|
|
const { POST } = await import('../../src/routes/api/agent/heartbeat/+server.js');
|
|
const cleanup = vi.spyOn(libraries, 'cleanupControllerLibraries').mockRejectedValue(new Error('Cleanup is independent of heartbeats'));
|
|
try {
|
|
for (let i = 0; i < 3; i++) {
|
|
const response = await POST({ request: new Request('https://controller.test/api/agent/heartbeat', {
|
|
method: 'POST', headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ inventory: { ...inventory, libraryCaches: [] } })
|
|
}) });
|
|
expect(response.status).toBe(200);
|
|
}
|
|
expect(cleanup).not.toHaveBeenCalled();
|
|
} finally { cleanup.mockRestore(); }
|
|
});
|
|
|
|
it.each([false, true])('keeps re-enabled hosts disabled until required files are verified, without waiting for the optional cache (supported: %s)', async supported => {
|
|
const t5 = await publish('t5');
|
|
const t6 = await publish('t6');
|
|
const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })),
|
|
capabilities: [...inventory.capabilities!, ...(supported ? ['plutonium.cache'] : [])] };
|
|
const id = enroll(readyInventory);
|
|
verified(id, t5); verified(id, t6);
|
|
const oldRuntime = hosts.queueHostOperation(id, 'runtime.pull', { image: runtime });
|
|
hosts.reportHostOperation(id, oldRuntime.id, { state: 'completed' });
|
|
heartbeat(id, readyInventory);
|
|
hosts.updateHost(id, { mode: 'disabled' });
|
|
const requested = hosts.updateHost(id, { mode: 'active' });
|
|
expect(requested).toMatchObject({ mode: 'disabled', readiness: 'preparing', enablingSince: expect.any(String) });
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
expect(() => hosts.requireReadyHost(id)).toThrow('before enabling');
|
|
preparation.reconcileHostPreparation(id);
|
|
const pull = operations(id)[0];
|
|
expect(pull.id).not.toBe(oldRuntime.id);
|
|
expect(pull.kind).toBe('runtime.pull');
|
|
expect(hosts.pendingHostOperations(id).map(operation => operation.id)).toEqual([pull.id]);
|
|
expect(() => hosts.queueHostOperation(id, 'instance.restart', {})).toThrow('disabled');
|
|
expect(hosts.updateHost(id, { mode: 'active' }).enablingSince).toBe(requested.enablingSince);
|
|
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)?.mode).toBe('disabled');
|
|
heartbeat(id, readyInventory);
|
|
for (let index = 0; index < 2; index++) {
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
expect(download.kind).toBe('library.download');
|
|
expect(hosts.pendingHostOperations(id).map(operation => operation.id)).toEqual([download.id]);
|
|
const version = [t5, t6].find(version => version.id === download.payload.libraryVersionId)!;
|
|
verified(id, version);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)?.mode).toBe('disabled');
|
|
heartbeat(id, readyInventory);
|
|
}
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)).toMatchObject({ mode: 'active', readiness: 'ready', enablingSince: null });
|
|
expect(operations(id).filter(operation => operation.kind === 'library.download')).toHaveLength(2);
|
|
expect(operations(id).filter(operation => operation.kind === 'plutonium.prepare')).toHaveLength(supported ? 1 : 0);
|
|
});
|
|
|
|
it('requires both games when enabling even if one library is already cached', async () => {
|
|
const version = await publish('t5');
|
|
const id = enroll({ runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] });
|
|
verified(id, version);
|
|
hosts.updateHost(id, { mode: 'disabled' });
|
|
hosts.updateHost(id, { mode: 'active' });
|
|
const status = preparation.reconcileHostPreparation(id);
|
|
expect(status.items).toContainEqual(expect.objectContaining({ label: 'Black Ops II (T6)', state: 'blocked' }));
|
|
expect(hosts.getHost(id)?.mode).toBe('disabled');
|
|
});
|
|
|
|
it('keeps a re-enabling host out of placement after setup failure or loss of contact', async () => {
|
|
const t5 = await publish('t5');
|
|
const t6 = await publish('t6');
|
|
const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })) };
|
|
const id = enroll(readyInventory);
|
|
verified(id, t5); verified(id, t6);
|
|
hosts.updateHost(id, { mode: 'disabled' });
|
|
hosts.updateHost(id, { mode: 'active' });
|
|
preparation.reconcileHostPreparation(id);
|
|
hosts.reportHostOperation(id, operations(id)[0].id, { state: 'failed', error: 'Registry unavailable' });
|
|
heartbeat(id, readyInventory);
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(() => hosts.requireReadyHost(id)).toThrow('before enabling');
|
|
const queued = operations(id).map(operation => operation.id);
|
|
vi.setSystemTime(Date.now() + 91_000);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)).toMatchObject({ mode: 'disabled', connected: false });
|
|
expect(operations(id).map(operation => operation.id)).toEqual(queued);
|
|
heartbeat(id, readyInventory);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)?.mode).toBe('disabled');
|
|
});
|
|
|
|
it('prepares an enrolled host through authenticated agent polls and heartbeats without manual setup requests', async () => {
|
|
const [{ POST: enrollAgent }, { GET: pollOperations }, { POST: reportOperation }, { POST: reportHeartbeat }] = await Promise.all([
|
|
import('../../src/routes/api/agent/enroll/+server.js'),
|
|
import('../../src/routes/api/agent/operations/+server.js'),
|
|
import('../../src/routes/api/agent/operations/[id]/+server.js'),
|
|
import('../../src/routes/api/agent/heartbeat/+server.js')
|
|
]);
|
|
const host = hosts.createHost({ name: 'Agent provisioning integration' });
|
|
const enrollment = await enrollAgent({ request: new Request('https://controller.test/api/agent/enroll', {
|
|
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ token: hosts.createEnrollment(host.id).token, inventory })
|
|
}) });
|
|
expect(enrollment.status).toBe(200);
|
|
const { credential } = await enrollment.json();
|
|
const request = (path: string, body?: object) => new Request(`https://controller.test/api/agent/${path}`, {
|
|
method: body ? 'POST' : 'GET',
|
|
headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' },
|
|
...(body ? { body: JSON.stringify(body) } : {})
|
|
});
|
|
expect(operations(host.id)).toEqual([]);
|
|
|
|
// Poll lanes only deliver queued work. One inventory heartbeat drives setup.
|
|
const reconcile = vi.spyOn(preparation, 'reconcileHostPreparation');
|
|
expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([]);
|
|
expect(reconcile).not.toHaveBeenCalled();
|
|
expect((await reportHeartbeat({ request: request('heartbeat', { inventory }) })).status).toBe(200);
|
|
expect(reconcile).toHaveBeenCalledOnce();
|
|
reconcile.mockRestore();
|
|
|
|
const firstPoll = await (await pollOperations({ request: request('operations') })).json();
|
|
expect(firstPoll.operations).toHaveLength(1);
|
|
const runtimeOperation = firstPoll.operations[0];
|
|
expect(runtimeOperation).toMatchObject({ kind: 'runtime.pull', payload: { image: runtime } });
|
|
expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([runtimeOperation]);
|
|
const version = await publish();
|
|
expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([runtimeOperation]);
|
|
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const completedRuntime = await reportOperation({
|
|
request: request(`operations/${runtimeOperation.id}`, { state: 'completed' }), params: { id: runtimeOperation.id }
|
|
});
|
|
expect(completedRuntime.status).toBe(200);
|
|
expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([]);
|
|
expect(operations(host.id)).toHaveLength(1);
|
|
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const preparedInventory = { ...inventory, runtimeImages: [runtime], libraryCaches: [] };
|
|
const runtimeHeartbeat = await reportHeartbeat({ request: request('heartbeat', { inventory: preparedInventory }) });
|
|
expect(runtimeHeartbeat.status).toBe(200);
|
|
expect(operations(host.id)).toHaveLength(2);
|
|
const download = operations(host.id)[0];
|
|
expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: version.id } });
|
|
expect((await reportHeartbeat({ request: request('heartbeat', { inventory: preparedInventory }) })).status).toBe(200);
|
|
expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([download]);
|
|
expect(operations(host.id)).toHaveLength(2);
|
|
|
|
vi.setSystemTime(Date.now() + 1);
|
|
expect((await reportOperation({
|
|
request: request(`operations/${download.id}`, { state: 'completed' }), params: { id: download.id }
|
|
})).status).toBe(200);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const finalHeartbeat = await reportHeartbeat({ request: request('heartbeat', { inventory: {
|
|
...preparedInventory,
|
|
libraries: [{ id: version.id, state: 'ready' }],
|
|
libraryCaches: [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]
|
|
} }) });
|
|
expect(finalHeartbeat.status).toBe(200);
|
|
expect(libraries.requireReadyLibrary(host.id, version.id).id).toBe(version.id);
|
|
expect(preparation.hostPreparationStatus(hosts.getHost(host.id)!).state).toBe('ready');
|
|
expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([]);
|
|
expect(operations(host.id)).toHaveLength(2);
|
|
});
|
|
|
|
it('presents preparation without queuing work and prepares the configured runtime even when another image exists', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: ['another:runtime'] });
|
|
const host = hosts.getHost(id)!;
|
|
const status = preparation.hostPreparationStatus(host);
|
|
expect(status.items.map(item => item.kind)).toEqual(['runtime', 'library']);
|
|
expect(preparation.presentHostPreparation(host)).toMatchObject({ id, preparation: status });
|
|
expect(operations(id)).toEqual([]);
|
|
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
|
const [operation] = operations(id);
|
|
expect(operation).toMatchObject({ kind: 'runtime.pull', payload: { image: runtime }, state: 'pending' });
|
|
expect(libraries.getHostLibraryCache(id, version.id)).toBeNull();
|
|
for (let poll = 0; poll < 4; poll++) {
|
|
heartbeat(id);
|
|
preparation.reconcileHostPreparation(id);
|
|
}
|
|
hosts.reportHostOperation(id, operation.id, { state: 'unknown' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id).map(item => item.id)).toEqual([operation.id]);
|
|
});
|
|
|
|
it('waits for fresh inventory after runtime completion before starting a library download', async () => {
|
|
const version = await publish();
|
|
const id = enroll();
|
|
preparation.reconcileHostPreparation(id);
|
|
const runtimeOperation = operations(id)[0];
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, runtimeOperation.id, { state: 'completed', result: { image: runtime } });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
|
|
heartbeat(id, { runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(2);
|
|
const download = operations(id)[0];
|
|
expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: version.id } });
|
|
verified(id, version);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
expect(operations(id)).toHaveLength(2);
|
|
});
|
|
|
|
it('always targets the latest verified T5 and T6 upload while newer uploads remain unfinished', async () => {
|
|
const old = await publish();
|
|
const latest = await publish();
|
|
const t6 = await publish('t6');
|
|
libraries.createLibraryUpload({
|
|
game: 't5',
|
|
files: [{ path: 'zone/game.ff', size: content.length, lastModified: 0 }]
|
|
});
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
const expected = new Map([latest, t6].map(version => [version.id, version]));
|
|
const downloaded: string[] = [];
|
|
for (let index = 0; index < expected.size; index++) {
|
|
preparation.reconcileHostPreparation(id);
|
|
const active = operations(id).filter(operation => !['completed', 'failed'].includes(operation.state));
|
|
expect(active).toHaveLength(1);
|
|
const operation = active[0];
|
|
const versionId = String(operation.payload.libraryVersionId);
|
|
expect(expected.has(versionId)).toBe(true);
|
|
expect(downloaded).not.toContain(versionId);
|
|
downloaded.push(versionId);
|
|
verified(id, expected.get(versionId)!);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, operation.id, { state: 'completed' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(index + 1);
|
|
heartbeat(id, { runtimeImages: [runtime], libraries: downloaded.map(id => ({ id, state: 'ready' })) });
|
|
}
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
expect(downloaded).toHaveLength(expected.size);
|
|
expect(downloaded).not.toContain(old.id);
|
|
});
|
|
|
|
it('picks up publication after enrollment and keeps verified historical versions intact', async () => {
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toEqual([]);
|
|
const old = await publish();
|
|
verified(id, old);
|
|
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(latest.id);
|
|
expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id);
|
|
expect(libraries.getHostLibraryCache(id, old.id)?.status).toBe('ready');
|
|
});
|
|
|
|
it('replaces an undelivered automatic download when newer files finish uploading', async () => {
|
|
const old = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const first = operations(id)[0];
|
|
expect(first.payload).toMatchObject({ libraryVersionId: old.id, automatic: true });
|
|
// The next heartbeat may discover a completed cache while this queued
|
|
// retry has still never been delivered. Keep that verified evidence.
|
|
verified(id, old);
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHostOperation(first.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('Superseded') });
|
|
expect(operations(id).filter(operation => operation.state === 'pending')).toMatchObject([
|
|
{ payload: { libraryVersionId: latest.id, automatic: true } }
|
|
]);
|
|
expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id);
|
|
});
|
|
|
|
it.each([false, true])('prepares queued libraries without pulling custom runtimes (automatic placement: %s)', async automatic => {
|
|
const old = await publish();
|
|
const image = 'ghcr.io/example/plutainer:queued-version';
|
|
const id = enroll({ runtimeImages: [runtime, image] });
|
|
const other = enroll({ runtimeImages: [runtime] });
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
for (let index = 0; index < 2; index++) {
|
|
expect(queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', libraryId: old.id,
|
|
hostId: automatic ? undefined : id, image, plutoKey: `saved-key-${index}` }).ok).toBe(true);
|
|
}
|
|
// Also cover pre-existing intents whose image is no longer cached.
|
|
heartbeat(id, { runtimeImages: [runtime] });
|
|
const latest = await publish();
|
|
const status = preparation.hostPreparationStatus(hosts.getHost(id)!);
|
|
expect(status.items.filter(item => item.label === image)).toHaveLength(0);
|
|
expect(status.items.filter(item => item.label.includes('Queued version'))).toHaveLength(1);
|
|
const otherStatus = preparation.hostPreparationStatus(hosts.getHost(other)!);
|
|
expect(otherStatus.items.some(item => item.label === image)).toBe(false);
|
|
expect(otherStatus.items.some(item => item.label.includes('Queued version'))).toBe(automatic);
|
|
expect(operations(id)).toEqual([]);
|
|
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: old.id, automatic: true } });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHostOperation(download.id)?.state).toBe('pending');
|
|
expect(operations(id)).toHaveLength(1);
|
|
preparation.reconcileHostPreparation(other);
|
|
expect([...operations(id), ...operations(other)].every(operation => operation.kind !== 'runtime.pull')).toBe(true);
|
|
|
|
verified(id, old);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
heartbeat(id, { runtimeImages: [runtime, image], libraries: [{ id: old.id, state: 'ready' }] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } });
|
|
});
|
|
|
|
it('retains an undelivered library download for an accepted intent until that intent is cancelled', async () => {
|
|
const old = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const launch = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', libraryId: old.id,
|
|
hostId: id, image: runtime, plutoKey: 'saved-key' });
|
|
expect(launch.ok).toBe(true);
|
|
if (!launch.ok) throw new Error(launch.message);
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHostOperation(download.id)?.state).toBe('pending');
|
|
expect(operations(id)).toHaveLength(1);
|
|
|
|
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(cancelCreation(launch.deploymentId)).toEqual(expect.any(String));
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHostOperation(download.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('Superseded') });
|
|
expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } });
|
|
});
|
|
|
|
it('finishes a delivered download before following a newer upload under a new operation ID', async () => {
|
|
const old = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const [first] = hosts.pendingHostOperations(id);
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(hosts.getHostOperation(first.id)?.state).toBe('pending');
|
|
verified(id, old);
|
|
hosts.reportHostOperation(id, first.id, { state: 'completed' });
|
|
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)[0]).toMatchObject({ state: 'pending', payload: { libraryVersionId: latest.id } });
|
|
expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id);
|
|
});
|
|
|
|
it.each(['draining', 'disabled'] as const)('does not start setup for a %s host', mode => {
|
|
const id = enroll();
|
|
hosts.updateHost(id, { mode });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it.each([
|
|
{ name: 'incompatible protocol', value: { protocolVersion: 2 } },
|
|
{ name: 'unavailable Docker', value: { dockerRunning: false } },
|
|
{ name: 'unwritable managed directories', value: { directoriesWritable: false } },
|
|
{ name: 'insufficient storage reserve', value: { storageAvailable: 0 } }
|
|
])('does not start setup with $name', ({ value }) => {
|
|
const id = enroll(value);
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it('does not start setup for unenrolled, disconnected, revoked, or decommissioning hosts', () => {
|
|
const unenrolled = hosts.createHost({ name: 'Not enrolled' }).id;
|
|
preparation.reconcileHostPreparation(unenrolled);
|
|
expect(operations(unenrolled)).toEqual([]);
|
|
const offline = enroll();
|
|
vi.setSystemTime(Date.now() + 91_000);
|
|
preparation.reconcileHostPreparation(offline);
|
|
expect(operations(offline)).toEqual([]);
|
|
const revoked = enroll();
|
|
hosts.revokeCredentials(revoked);
|
|
preparation.reconcileHostPreparation(revoked);
|
|
expect(operations(revoked)).toEqual([]);
|
|
const deleting = enroll();
|
|
hosts.deleteHost(deleting);
|
|
preparation.reconcileHostPreparation(deleting);
|
|
expect(operations(deleting)).toEqual([]);
|
|
});
|
|
|
|
it('retries runtime failures with exponential delays capped at fifteen minutes', () => {
|
|
const id = enroll();
|
|
preparation.reconcileHostPreparation(id);
|
|
for (const [attempt, delay] of [60_000, 120_000, 240_000, 480_000, 900_000, 900_000].entries()) {
|
|
const operation = operations(id)[0];
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Registry unavailable' });
|
|
const failedAt = Date.now();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(attempt + 1);
|
|
vi.setSystemTime(failedAt + delay - 2);
|
|
heartbeat(id);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(attempt + 1);
|
|
vi.setSystemTime(failedAt + delay);
|
|
heartbeat(id);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(attempt + 2);
|
|
expect(operations(id)[0]).toMatchObject({ kind: 'runtime.pull', state: 'pending' });
|
|
}
|
|
});
|
|
|
|
it('blocks a library that cannot fit while preserving the host free-space reserve', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes - 1 });
|
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(operations(id)).toEqual([]);
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
|
});
|
|
|
|
it('returns an active download even when its in-progress bytes have consumed free space', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
const operation = libraries.queueLibraryDownload(id, version.id);
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: hosts.getHost(id)!.minimumFreeBytes });
|
|
expect(libraries.queueLibraryDownload(id, version.id).id).toBe(operation.id);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
});
|
|
|
|
it('prepares T6 when the latest T5 upload cannot fit', async () => {
|
|
const oversized = await publish();
|
|
const small = await publish('t6', 'small');
|
|
const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_010 });
|
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
|
const status = preparation.reconcileHostPreparation(id);
|
|
expect(status.items).toEqual(expect.arrayContaining([
|
|
expect.objectContaining({ label: 'Black Ops I (T5)', state: 'blocked' }),
|
|
expect.objectContaining({ label: 'Black Ops II (T6)', state: 'preparing' })
|
|
]));
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(small.id);
|
|
expect(libraries.getHostLibraryCache(id, oversized.id)).toBeNull();
|
|
});
|
|
|
|
it('does not count partial bytes from a different manifest toward download storage', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
libraries.updateHostLibraryCaches(id, [{
|
|
libraryVersionId: version.id, status: 'failed', bytesDownloaded: 80, manifestSha256: '0'.repeat(64)
|
|
}]);
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20 });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(() => libraries.queueLibraryDownload(id, version.id)).toThrow(/insufficient storage/);
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it('resumes a failed verified-manifest download using only its remaining storage', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const operation = operations(id)[0];
|
|
libraries.updateHostLibraryCaches(id, [{
|
|
libraryVersionId: version.id, status: 'downloading', bytesDownloaded: 80, manifestSha256: version.manifestSha256
|
|
}]);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Connection reset' });
|
|
const failedAt = Date.now();
|
|
const free = hosts.getHost(id)!.minimumFreeBytes + version.sizeBytes - 80;
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
vi.setSystemTime(failedAt + 60_000);
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
|
expect(operations(id)).toHaveLength(2);
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
|
expect(libraries.getHostLibraryCache(id, version.id)?.bytesDownloaded).toBe(80);
|
|
});
|
|
});
|
|
|
|
describe('optional host Plutonium cache', () => {
|
|
const capable = { capabilities: ['host.decommission', 'plutonium.cache'] };
|
|
async function release(revision: number, overrides: { checkedAt?: string; error?: string | null } = {}) {
|
|
const { writeSetting } = await import('../../src/lib/server/maintenance/settings.js');
|
|
writeSetting('plutonium_release', { revision, checkedAt: new Date().toISOString(), error: null, ...overrides });
|
|
}
|
|
|
|
it.each(['pending', 'accepted', 'running', 'unknown'] as const)('waits to warm while a deployment is %s', state => {
|
|
const ready = { ...capable, runtimeImages: [runtime], libraries: [{ id: 'game-files', state: 'ready' as const }] };
|
|
const id = enroll(ready);
|
|
const deployment = hosts.queueHostOperation(id, 'instance.create', {});
|
|
if (state !== 'pending') hosts.reportHostOperation(id, deployment.id, { state });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(true);
|
|
expect(operations(id)).toEqual([]);
|
|
hosts.reportHostOperation(id, deployment.id, { state: 'completed' });
|
|
heartbeat(id, ready);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toMatchObject([{ kind: 'plutonium.prepare' }]);
|
|
});
|
|
|
|
it('delivers newer deployments before queued cache work and before a cache replay', () => {
|
|
const id = enroll({ ...capable, runtimeImages: [runtime], libraries: [{ id: 'game-files', state: 'ready' }] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const cache = operations(id)[0];
|
|
const deployment = hosts.queueHostOperation(id, 'instance.create', {});
|
|
expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([deployment.id, cache.id]);
|
|
hosts.reportHostOperation(id, cache.id, { state: 'running' });
|
|
const change = hosts.queueHostOperation(id, 'files.stage', {});
|
|
const delivery = hosts.pendingHostOperations(id, 'write').map(operation => operation.id);
|
|
expect(delivery.at(-1)).toBe(cache.id);
|
|
expect(delivery).toContain(deployment.id);
|
|
expect(delivery).toContain(change.id);
|
|
});
|
|
|
|
it('signals deferral on heartbeats for mutations while ignoring cache work and all observation lanes', async () => {
|
|
const host = hosts.createHost({ name: 'Cache yielding host' });
|
|
const ready = { ...inventory, ...capable, runtimeImages: [runtime], libraries: [{ id: 'game-files', state: 'ready' as const }] };
|
|
const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, ready);
|
|
preparation.reconcileHostPreparation(host.id);
|
|
const cache = operations(host.id)[0];
|
|
hosts.reportHostOperation(host.id, cache.id, { state: 'running' });
|
|
for (const kind of ['host.inspect', 'instance.inspect', 'rcon.status-batch', 'files.read', 'files.list', 'files.stat', 'files.manifest', 'logs.read']) {
|
|
hosts.queueHostOperation(host.id, kind, {});
|
|
}
|
|
for (const command of ['status', 'getstatus', 'version']) hosts.queueHostOperation(host.id, 'rcon.execute', { command });
|
|
// Work belonging to another host must not pause this one's cache.
|
|
const other = enroll(ready);
|
|
hosts.queueHostOperation(other, 'instance.restart', {});
|
|
const { POST } = await import('../../src/routes/api/agent/heartbeat/+server.js');
|
|
const poll = async () => {
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const response = await POST({ request: new Request('https://controller.test/api/agent/heartbeat', {
|
|
method: 'POST', headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ inventory: ready })
|
|
}) });
|
|
expect(response.status).toBe(200);
|
|
return response.json();
|
|
};
|
|
expect(await poll()).toMatchObject({ deferPlutoniumPreparation: false });
|
|
const change = hosts.queueHostOperation(host.id, 'rcon.execute', { command: 'map_restart' });
|
|
expect(await poll()).toMatchObject({ deferPlutoniumPreparation: true });
|
|
hosts.reportHostOperation(host.id, change.id, { state: 'failed', error: 'Command rejected' });
|
|
expect(await poll()).toMatchObject({ deferPlutoniumPreparation: false });
|
|
expect(operations(host.id).filter(operation => operation.kind === 'plutonium.prepare').map(operation => operation.id)).toEqual([cache.id]);
|
|
});
|
|
|
|
it('lets newly published game files interrupt cache refresh, then resumes the same cache operation', async () => {
|
|
const version = await publish();
|
|
const ready = { ...capable, runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' as const }],
|
|
plutoniumCache: { revision: 5354, bytes: 200 } };
|
|
const id = enroll(ready);
|
|
verified(id, version);
|
|
await release(5355);
|
|
preparation.reconcileHostPreparation(id);
|
|
const cache = operations(id)[0];
|
|
expect(cache).toMatchObject({ kind: 'plutonium.prepare', payload: { revision: 5355 } });
|
|
hosts.reportHostOperation(id, cache.id, { state: 'running' });
|
|
const latest = await publish();
|
|
heartbeat(id, ready);
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } });
|
|
expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(true);
|
|
expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([download.id, cache.id]);
|
|
verified(id, latest);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
heartbeat(id, { ...ready, libraries: [{ id: latest.id, state: 'ready' }] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(false);
|
|
expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([cache.id]);
|
|
expect(operations(id).filter(operation => operation.kind === 'plutonium.prepare')).toHaveLength(1);
|
|
});
|
|
|
|
it('warms only after required files, preserves launch readiness, and waits for fresh cache inventory', async () => {
|
|
const version = await publish();
|
|
await release(5354);
|
|
const id = enroll(capable);
|
|
preparation.reconcileHostPreparation(id);
|
|
const pull = operations(id)[0];
|
|
expect(pull.kind).toBe('runtime.pull');
|
|
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
|
|
heartbeat(id, { ...capable, runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
expect(download.kind).toBe('library.download');
|
|
verified(id, version);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
const ready = { ...capable, runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' as const }] };
|
|
heartbeat(id, ready);
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
const cache = operations(id)[0];
|
|
expect(cache).toMatchObject({ kind: 'plutonium.prepare', payload: {
|
|
revision: 5354, minimumFreeBytes: hosts.getHost(id)!.minimumFreeBytes, automatic: true
|
|
} });
|
|
expect(hosts.requireReadyHost(id).readiness).toBe('ready');
|
|
expect(hosts.pendingHostOperations(id, 'read')).toEqual([]);
|
|
expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([cache.id]);
|
|
expect(hosts.listHostOperationHistory(1, 'important', { host: id }).operations[0]).toMatchObject({
|
|
id: cache.id, routine: false, targetPath: 'Production revision r5354'
|
|
});
|
|
preparation.reconcileHostPreparation(id);
|
|
hosts.reportHostOperation(id, cache.id, { state: 'completed' });
|
|
const pendingInventory = preparation.reconcileHostPreparation(id);
|
|
expect(pendingInventory.items.find(item => item.kind === 'plutonium')).toMatchObject({ state: 'preparing' });
|
|
expect(operations(id)).toHaveLength(3);
|
|
heartbeat(id, { ...ready, plutoniumCache: { revision: 5354, bytes: 25_000 } });
|
|
expect(preparation.reconcileHostPreparation(id).items.find(item => item.kind === 'plutonium')).toMatchObject({
|
|
state: 'ready', detail: expect.stringContaining('r5354')
|
|
});
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(3);
|
|
});
|
|
|
|
it('refreshes for newer production releases and reconciles an interrupted refresh without duplicates', async () => {
|
|
const ready = { ...capable, runtimeImages: [runtime], plutoniumCache: { revision: 5354, bytes: 25_000 } };
|
|
const id = enroll(ready);
|
|
await release(5354);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toEqual([]);
|
|
await release(5355);
|
|
preparation.reconcileHostPreparation(id);
|
|
const cache = operations(id)[0];
|
|
expect(cache.payload.revision).toBe(5355);
|
|
hosts.reportHostOperation(id, cache.id, { state: 'unknown' });
|
|
await release(5356);
|
|
heartbeat(id, ready);
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
expect(preparation.reconcileHostPreparation(id).items.find(item => item.kind === 'plutonium')).toMatchObject({
|
|
state: 'preparing', detail: expect.stringContaining('reconcile interrupted')
|
|
});
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(hosts.pendingHostOperations(id, 'write').map(operation => operation.id)).toEqual([cache.id]);
|
|
hosts.reportHostOperation(id, cache.id, { state: 'completed' });
|
|
heartbeat(id, { ...ready, plutoniumCache: { revision: 5356, bytes: 26_000 } });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
await release(5355);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
});
|
|
|
|
it.each(['missing', 'stale', 'failed'] as const)('uses the agent production manifest when release evidence is %s', async evidence => {
|
|
if (evidence === 'stale') await release(5354, { checkedAt: new Date(Date.now() - 60 * 60_000).toISOString() });
|
|
if (evidence === 'failed') await release(5354, { error: 'Manifest unavailable' });
|
|
const id = enroll({ ...capable, runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)[0]).toMatchObject({ kind: 'plutonium.prepare', payload: { revision: 0 } });
|
|
expect(hosts.listHostOperationHistory(1, 'important', { host: id }).operations[0].targetPath).toBe('Latest production revision');
|
|
const cached = enroll({ ...capable, runtimeImages: [runtime], plutoniumCache: { revision: 5353, bytes: 200 } });
|
|
preparation.reconcileHostPreparation(cached);
|
|
expect(operations(cached)).toEqual([]);
|
|
});
|
|
|
|
it('leaves agents without the cache capability unchanged', async () => {
|
|
await release(5355);
|
|
const id = enroll({ runtimeImages: [runtime], plutoniumCache: { revision: 5354, bytes: 200 } });
|
|
expect(preparation.reconcileHostPreparation(id).items.some(item => item.kind === 'plutonium')).toBe(false);
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it('backs off failed cache work without blocking launch readiness or newly published game files', async () => {
|
|
const version = await publish();
|
|
const ready = { ...capable, runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' as const }] };
|
|
const id = enroll(ready);
|
|
verified(id, version);
|
|
await release(5354);
|
|
preparation.reconcileHostPreparation(id);
|
|
const cache = operations(id)[0];
|
|
hosts.reportHostOperation(id, cache.id, { state: 'failed', error: 'Plutonium CDN unavailable' });
|
|
const failedAt = Date.now();
|
|
heartbeat(id, ready);
|
|
const status = preparation.reconcileHostPreparation(id);
|
|
expect(status).toMatchObject({ state: 'ready' });
|
|
expect(status.items.find(item => item.kind === 'plutonium')).toMatchObject({
|
|
state: 'blocked', detail: expect.stringContaining('Automatic retry')
|
|
});
|
|
expect(hosts.requireReadyHost(id).readiness).toBe('ready');
|
|
expect(operations(id)).toHaveLength(1);
|
|
// A changed or expired release must not bypass a failed cache's backoff.
|
|
await release(5355);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } });
|
|
verified(id, latest);
|
|
hosts.reportHostOperation(id, operations(id)[0].id, { state: 'completed' });
|
|
heartbeat(id, ready);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(2);
|
|
vi.setSystemTime(failedAt + 60_000);
|
|
heartbeat(id, ready);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(3);
|
|
expect(operations(id)[0]).toMatchObject({ kind: 'plutonium.prepare', payload: { revision: 5355 } });
|
|
});
|
|
|
|
it('waits for required library capacity before consuming space for the optional cache', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ ...capable, runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes - 1 });
|
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it.each([null, [], {}, { revision: 0, bytes: 1 }, { revision: 1.5, bytes: 1 }, { revision: 1, bytes: -1 }, { revision: 1, bytes: 0.5 }])(
|
|
'rejects invalid cache inventory %j', cache => {
|
|
const id = enroll();
|
|
expect(() => heartbeat(id, { plutoniumCache: cache as HostInventory['plutoniumCache'] })).toThrow('Invalid Plutonium cache inventory');
|
|
}
|
|
);
|
|
|
|
it('retains only revision and byte count from cache inventory and clears an omitted cache', () => {
|
|
const id = enroll({ plutoniumCache: { revision: 5354, bytes: 0, localPath: '/private/cache' } as HostInventory['plutoniumCache'] });
|
|
expect(hosts.getHost(id)?.inventory.plutoniumCache).toEqual({ revision: 5354, bytes: 0 });
|
|
heartbeat(id);
|
|
expect(hosts.getHost(id)?.inventory.plutoniumCache).toBeUndefined();
|
|
});
|
|
});
|