import { createHash } from 'node:crypto'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { GameLibraryVersion } from '../../src/lib/game-files.js'; import type { HostInventory } from '../../src/lib/hosts.js'; import { newFixture, type Fixture } from '../helpers/fixture.js'; let fixture: Fixture; let hosts: typeof import('../../src/lib/server/hosts/index.js'); let libraries: typeof import('../../src/lib/server/game-files/index.js'); let preparation: typeof import('../../src/lib/server/hosts/preparation.js'); const runtime = 'ghcr.io/example/plutainer:provisioning-test'; const initialTime = Date.parse('2026-10-04T12:00:00Z'); const content = 'a'.repeat(100); const inventory: HostInventory = { hostname: 'game-host', protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3, runtimeImages: [], libraries: [], capabilities: ['host.decommission'] }; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); vi.stubEnv('PLUTAINER_IMAGE', runtime); vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'game-files')); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(initialTime); hosts = await import('../../src/lib/server/hosts/index.js'); libraries = await import('../../src/lib/server/game-files/index.js'); preparation = await import('../../src/lib/server/hosts/preparation.js'); }); afterEach(async () => { vi.useRealTimers(); vi.unstubAllEnvs(); await fixture.cleanup(); }); function enroll(overrides: HostInventory = {}) { const host = hosts.createHost({ name: 'Automatic game host' }); hosts.enrollHost(hosts.createEnrollment(host.id).token, { ...inventory, ...overrides }); return host.id; } function heartbeat(id: string, overrides: HostInventory = {}) { vi.setSystemTime(Date.now() + 1); return hosts.heartbeat(id, { ...inventory, ...overrides }); } function operations(id: string) { return hosts.listHostOperations(id).filter(operation => ['runtime.pull', 'library.download'].includes(operation.kind)); } async function publish(game: 't5' | 't6' = 't5', bytes = content) { vi.setSystemTime(Date.now() + 1); const upload = libraries.createLibraryUpload({ game, files: [{ path: 'zone/game.ff', size: bytes.length, lastModified: 0 }] }); await libraries.uploadBrowserFile(upload.id, 'zone/game.ff', { offset: 0, size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') }, new Response(bytes).body); return libraries.publishBrowserUpload(upload.id); } function verified(id: string, version: GameLibraryVersion) { libraries.updateHostLibraryCaches(id, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]); } describe('automatic host preparation', () => { it('keeps re-enabled hosts disabled until a fresh runtime pull and both game libraries are verified', async () => { const t5 = await publish('t5'); const t6 = await publish('t6'); const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })) }; const id = enroll(readyInventory); verified(id, t5); verified(id, t6); const oldRuntime = hosts.queueHostOperation(id, 'runtime.pull', { image: runtime }); hosts.reportHostOperation(id, oldRuntime.id, { state: 'completed' }); heartbeat(id, readyInventory); hosts.updateHost(id, { mode: 'disabled' }); const requested = hosts.updateHost(id, { mode: 'active' }); expect(requested).toMatchObject({ mode: 'disabled', readiness: 'preparing', enablingSince: expect.any(String) }); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); expect(() => hosts.requireReadyHost(id)).toThrow('before enabling'); preparation.reconcileHostPreparation(id); const pull = operations(id)[0]; expect(pull.id).not.toBe(oldRuntime.id); expect(pull.kind).toBe('runtime.pull'); expect(hosts.pendingHostOperations(id).map(operation => operation.id)).toEqual([pull.id]); expect(() => hosts.queueHostOperation(id, 'instance.restart', {})).toThrow('disabled'); expect(hosts.updateHost(id, { mode: 'active' }).enablingSince).toBe(requested.enablingSince); hosts.reportHostOperation(id, pull.id, { state: 'completed' }); preparation.reconcileHostPreparation(id); expect(hosts.getHost(id)?.mode).toBe('disabled'); heartbeat(id, readyInventory); for (let index = 0; index < 2; index++) { preparation.reconcileHostPreparation(id); const download = operations(id)[0]; expect(download.kind).toBe('library.download'); expect(hosts.pendingHostOperations(id).map(operation => operation.id)).toEqual([download.id]); const version = [t5, t6].find(version => version.id === download.payload.libraryVersionId)!; verified(id, version); hosts.reportHostOperation(id, download.id, { state: 'completed' }); preparation.reconcileHostPreparation(id); expect(hosts.getHost(id)?.mode).toBe('disabled'); heartbeat(id, readyInventory); } preparation.reconcileHostPreparation(id); expect(hosts.getHost(id)).toMatchObject({ mode: 'active', readiness: 'ready', enablingSince: null }); expect(operations(id).filter(operation => operation.kind === 'library.download')).toHaveLength(2); }); it('requires both games when enabling even if one library is already cached', async () => { const version = await publish('t5'); const id = enroll({ runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] }); verified(id, version); hosts.updateHost(id, { mode: 'disabled' }); hosts.updateHost(id, { mode: 'active' }); const status = preparation.reconcileHostPreparation(id); expect(status.items).toContainEqual(expect.objectContaining({ label: 'Black Ops II (T6)', state: 'blocked' })); expect(hosts.getHost(id)?.mode).toBe('disabled'); }); it('keeps a re-enabling host out of placement after setup failure or loss of contact', async () => { const t5 = await publish('t5'); const t6 = await publish('t6'); const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })) }; const id = enroll(readyInventory); verified(id, t5); verified(id, t6); hosts.updateHost(id, { mode: 'disabled' }); hosts.updateHost(id, { mode: 'active' }); preparation.reconcileHostPreparation(id); hosts.reportHostOperation(id, operations(id)[0].id, { state: 'failed', error: 'Registry unavailable' }); heartbeat(id, readyInventory); expect(preparation.reconcileHostPreparation(id).state).toBe('blocked'); expect(() => hosts.requireReadyHost(id)).toThrow('before enabling'); const queued = operations(id).map(operation => operation.id); vi.setSystemTime(Date.now() + 91_000); preparation.reconcileHostPreparation(id); expect(hosts.getHost(id)).toMatchObject({ mode: 'disabled', connected: false }); expect(operations(id).map(operation => operation.id)).toEqual(queued); heartbeat(id, readyInventory); preparation.reconcileHostPreparation(id); expect(hosts.getHost(id)?.mode).toBe('disabled'); }); it('prepares an enrolled host through authenticated agent polls and heartbeats without manual setup requests', async () => { const [{ POST: enrollAgent }, { GET: pollOperations }, { POST: reportOperation }, { POST: reportHeartbeat }] = await Promise.all([ import('../../src/routes/api/agent/enroll/+server.js'), import('../../src/routes/api/agent/operations/+server.js'), import('../../src/routes/api/agent/operations/[id]/+server.js'), import('../../src/routes/api/agent/heartbeat/+server.js') ]); const host = hosts.createHost({ name: 'Agent provisioning integration' }); const enrollment = await enrollAgent({ request: new Request('https://controller.test/api/agent/enroll', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ token: hosts.createEnrollment(host.id).token, inventory }) }) }); expect(enrollment.status).toBe(200); const { credential } = await enrollment.json(); const request = (path: string, body?: object) => new Request(`https://controller.test/api/agent/${path}`, { method: body ? 'POST' : 'GET', headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' }, ...(body ? { body: JSON.stringify(body) } : {}) }); expect(operations(host.id)).toEqual([]); const firstPoll = await (await pollOperations({ request: request('operations') })).json(); expect(firstPoll.operations).toHaveLength(1); const runtimeOperation = firstPoll.operations[0]; expect(runtimeOperation).toMatchObject({ kind: 'runtime.pull', payload: { image: runtime } }); expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([runtimeOperation]); const version = await publish(); expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([runtimeOperation]); vi.setSystemTime(Date.now() + 1); const completedRuntime = await reportOperation({ request: request(`operations/${runtimeOperation.id}`, { state: 'completed' }), params: { id: runtimeOperation.id } }); expect(completedRuntime.status).toBe(200); expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([]); expect(operations(host.id)).toHaveLength(1); vi.setSystemTime(Date.now() + 1); const preparedInventory = { ...inventory, runtimeImages: [runtime], libraryCaches: [] }; const runtimeHeartbeat = await reportHeartbeat({ request: request('heartbeat', { inventory: preparedInventory }) }); expect(runtimeHeartbeat.status).toBe(200); expect(operations(host.id)).toHaveLength(2); const download = operations(host.id)[0]; expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: version.id } }); expect((await reportHeartbeat({ request: request('heartbeat', { inventory: preparedInventory }) })).status).toBe(200); expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([download]); expect(operations(host.id)).toHaveLength(2); vi.setSystemTime(Date.now() + 1); expect((await reportOperation({ request: request(`operations/${download.id}`, { state: 'completed' }), params: { id: download.id } })).status).toBe(200); vi.setSystemTime(Date.now() + 1); const finalHeartbeat = await reportHeartbeat({ request: request('heartbeat', { inventory: { ...preparedInventory, libraries: [{ id: version.id, state: 'ready' }], libraryCaches: [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }] } }) }); expect(finalHeartbeat.status).toBe(200); expect(libraries.requireReadyLibrary(host.id, version.id).id).toBe(version.id); expect(preparation.hostPreparationStatus(hosts.getHost(host.id)!).state).toBe('ready'); expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([]); expect(operations(host.id)).toHaveLength(2); }); it('presents preparation without queuing work and prepares the configured runtime even when another image exists', async () => { const version = await publish(); const id = enroll({ runtimeImages: ['another:runtime'] }); const host = hosts.getHost(id)!; const status = preparation.hostPreparationStatus(host); expect(status.items.map(item => item.kind)).toEqual(['runtime', 'library']); expect(preparation.presentHostPreparation(host)).toMatchObject({ id, preparation: status }); expect(operations(id)).toEqual([]); expect(preparation.reconcileHostPreparation(id).state).toBe('preparing'); const [operation] = operations(id); expect(operation).toMatchObject({ kind: 'runtime.pull', payload: { image: runtime }, state: 'pending' }); expect(libraries.getHostLibraryCache(id, version.id)).toBeNull(); for (let poll = 0; poll < 4; poll++) { heartbeat(id); preparation.reconcileHostPreparation(id); } hosts.reportHostOperation(id, operation.id, { state: 'unknown' }); preparation.reconcileHostPreparation(id); expect(operations(id).map(item => item.id)).toEqual([operation.id]); }); it('waits for fresh inventory after runtime completion before starting a library download', async () => { const version = await publish(); const id = enroll(); preparation.reconcileHostPreparation(id); const runtimeOperation = operations(id)[0]; vi.setSystemTime(Date.now() + 1); hosts.reportHostOperation(id, runtimeOperation.id, { state: 'completed', result: { image: runtime } }); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(1); heartbeat(id, { runtimeImages: [runtime] }); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(2); const download = operations(id)[0]; expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: version.id } }); verified(id, version); hosts.reportHostOperation(id, download.id, { state: 'completed' }); heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] }); expect(preparation.reconcileHostPreparation(id).state).toBe('ready'); expect(operations(id)).toHaveLength(2); }); it('always targets the latest verified T5 and T6 upload while newer uploads remain unfinished', async () => { const old = await publish(); const latest = await publish(); const t6 = await publish('t6'); libraries.createLibraryUpload({ game: 't5', files: [{ path: 'zone/game.ff', size: content.length, lastModified: 0 }] }); const id = enroll({ runtimeImages: [runtime] }); const expected = new Map([latest, t6].map(version => [version.id, version])); const downloaded: string[] = []; for (let index = 0; index < expected.size; index++) { preparation.reconcileHostPreparation(id); const active = operations(id).filter(operation => !['completed', 'failed'].includes(operation.state)); expect(active).toHaveLength(1); const operation = active[0]; const versionId = String(operation.payload.libraryVersionId); expect(expected.has(versionId)).toBe(true); expect(downloaded).not.toContain(versionId); downloaded.push(versionId); verified(id, expected.get(versionId)!); vi.setSystemTime(Date.now() + 1); hosts.reportHostOperation(id, operation.id, { state: 'completed' }); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(index + 1); heartbeat(id, { runtimeImages: [runtime], libraries: downloaded.map(id => ({ id, state: 'ready' })) }); } expect(preparation.reconcileHostPreparation(id).state).toBe('ready'); expect(downloaded).toHaveLength(expected.size); expect(downloaded).not.toContain(old.id); }); it('picks up publication after enrollment and keeps verified historical versions intact', async () => { const id = enroll({ runtimeImages: [runtime] }); preparation.reconcileHostPreparation(id); expect(operations(id)).toEqual([]); const old = await publish(); verified(id, old); heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] }); expect(preparation.reconcileHostPreparation(id).state).toBe('ready'); const latest = await publish(); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(1); expect(operations(id)[0].payload.libraryVersionId).toBe(latest.id); expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id); expect(libraries.getHostLibraryCache(id, old.id)?.status).toBe('ready'); }); it('replaces an undelivered automatic download when newer files finish uploading', async () => { const old = await publish(); const id = enroll({ runtimeImages: [runtime] }); preparation.reconcileHostPreparation(id); const first = operations(id)[0]; expect(first.payload).toMatchObject({ libraryVersionId: old.id, automatic: true }); // The next heartbeat may discover a completed cache while this queued // retry has still never been delivered. Keep that verified evidence. verified(id, old); const latest = await publish(); preparation.reconcileHostPreparation(id); expect(hosts.getHostOperation(first.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('Superseded') }); expect(operations(id).filter(operation => operation.state === 'pending')).toMatchObject([ { payload: { libraryVersionId: latest.id, automatic: true } } ]); expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id); }); it.each([false, true])('prepares queued runtime and library versions after newer publication (automatic placement: %s)', async automatic => { const old = await publish(); const id = enroll({ runtimeImages: [runtime] }); const other = enroll({ runtimeImages: [runtime] }); const image = 'ghcr.io/example/plutainer:queued-version'; const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); for (let index = 0; index < 2; index++) { expect(queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', libraryId: old.id, hostId: automatic ? undefined : id, image, plutoKey: 'saved-key' }).ok).toBe(true); } const latest = await publish(); const status = preparation.hostPreparationStatus(hosts.getHost(id)!); expect(status.items.filter(item => item.label === image)).toHaveLength(1); expect(status.items.filter(item => item.label.includes('Queued version'))).toHaveLength(1); const otherStatus = preparation.hostPreparationStatus(hosts.getHost(other)!); expect(otherStatus.items.some(item => item.label === image)).toBe(automatic); expect(otherStatus.items.some(item => item.label.includes('Queued version'))).toBe(automatic); expect(operations(id)).toEqual([]); preparation.reconcileHostPreparation(id); const pull = operations(id)[0]; expect(pull).toMatchObject({ kind: 'runtime.pull', payload: { image } }); vi.setSystemTime(Date.now() + 1); hosts.reportHostOperation(id, pull.id, { state: 'completed' }); heartbeat(id, { runtimeImages: [runtime, image] }); preparation.reconcileHostPreparation(id); const download = operations(id)[0]; expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: old.id, automatic: true } }); preparation.reconcileHostPreparation(id); expect(hosts.getHostOperation(download.id)?.state).toBe('pending'); expect(operations(id)).toHaveLength(2); verified(id, old); vi.setSystemTime(Date.now() + 1); hosts.reportHostOperation(id, download.id, { state: 'completed' }); heartbeat(id, { runtimeImages: [runtime, image], libraries: [{ id: old.id, state: 'ready' }] }); preparation.reconcileHostPreparation(id); expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } }); }); it('retains an undelivered library download for an accepted intent until that intent is cancelled', async () => { const old = await publish(); const id = enroll({ runtimeImages: [runtime] }); preparation.reconcileHostPreparation(id); const download = operations(id)[0]; const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js'); const launch = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', libraryId: old.id, hostId: id, image: runtime, plutoKey: 'saved-key' }); expect(launch.ok).toBe(true); if (!launch.ok) throw new Error(launch.message); const latest = await publish(); preparation.reconcileHostPreparation(id); expect(hosts.getHostOperation(download.id)?.state).toBe('pending'); expect(operations(id)).toHaveLength(1); const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); expect(cancelCreation(launch.deploymentId)).toEqual(expect.any(String)); preparation.reconcileHostPreparation(id); expect(hosts.getHostOperation(download.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('Superseded') }); expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } }); }); it('finishes a delivered download before following a newer upload under a new operation ID', async () => { const old = await publish(); const id = enroll({ runtimeImages: [runtime] }); preparation.reconcileHostPreparation(id); const [first] = hosts.pendingHostOperations(id); const latest = await publish(); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(1); expect(hosts.getHostOperation(first.id)?.state).toBe('pending'); verified(id, old); hosts.reportHostOperation(id, first.id, { state: 'completed' }); heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] }); preparation.reconcileHostPreparation(id); expect(operations(id)[0]).toMatchObject({ state: 'pending', payload: { libraryVersionId: latest.id } }); expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id); }); it.each(['draining', 'disabled'] as const)('does not start setup for a %s host', mode => { const id = enroll(); hosts.updateHost(id, { mode }); preparation.reconcileHostPreparation(id); expect(operations(id)).toEqual([]); }); it.each([ { name: 'incompatible protocol', value: { protocolVersion: 2 } }, { name: 'unavailable Docker', value: { dockerRunning: false } }, { name: 'unwritable managed directories', value: { directoriesWritable: false } }, { name: 'insufficient storage reserve', value: { storageAvailable: 0 } } ])('does not start setup with $name', ({ value }) => { const id = enroll(value); expect(preparation.reconcileHostPreparation(id).state).toBe('blocked'); expect(operations(id)).toEqual([]); }); it('does not start setup for unenrolled, disconnected, revoked, or decommissioning hosts', () => { const unenrolled = hosts.createHost({ name: 'Not enrolled' }).id; preparation.reconcileHostPreparation(unenrolled); expect(operations(unenrolled)).toEqual([]); const offline = enroll(); vi.setSystemTime(Date.now() + 91_000); preparation.reconcileHostPreparation(offline); expect(operations(offline)).toEqual([]); const revoked = enroll(); hosts.revokeCredentials(revoked); preparation.reconcileHostPreparation(revoked); expect(operations(revoked)).toEqual([]); const deleting = enroll(); hosts.deleteHost(deleting); preparation.reconcileHostPreparation(deleting); expect(operations(deleting)).toEqual([]); }); it('retries runtime failures with exponential delays capped at fifteen minutes', () => { const id = enroll(); preparation.reconcileHostPreparation(id); for (const [attempt, delay] of [60_000, 120_000, 240_000, 480_000, 900_000, 900_000].entries()) { const operation = operations(id)[0]; vi.setSystemTime(Date.now() + 1); hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Registry unavailable' }); const failedAt = Date.now(); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(attempt + 1); vi.setSystemTime(failedAt + delay - 2); heartbeat(id); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(attempt + 1); vi.setSystemTime(failedAt + delay); heartbeat(id); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(attempt + 2); expect(operations(id)[0]).toMatchObject({ kind: 'runtime.pull', state: 'pending' }); } }); it('blocks a library that cannot fit while preserving the host free-space reserve', async () => { const version = await publish(); const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes - 1 }); hosts.updateHost(id, { minimumFreeBytes: 1_000 }); expect(preparation.reconcileHostPreparation(id).state).toBe('blocked'); expect(operations(id)).toEqual([]); heartbeat(id, { runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes }); expect(preparation.reconcileHostPreparation(id).state).toBe('preparing'); expect(operations(id)[0].payload.libraryVersionId).toBe(version.id); }); it('returns an active download even when its in-progress bytes have consumed free space', async () => { const version = await publish(); const id = enroll({ runtimeImages: [runtime] }); const operation = libraries.queueLibraryDownload(id, version.id); heartbeat(id, { runtimeImages: [runtime], storageAvailable: hosts.getHost(id)!.minimumFreeBytes }); expect(libraries.queueLibraryDownload(id, version.id).id).toBe(operation.id); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(1); }); it('prepares T6 when the latest T5 upload cannot fit', async () => { const oversized = await publish(); const small = await publish('t6', 'small'); const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_010 }); hosts.updateHost(id, { minimumFreeBytes: 1_000 }); const status = preparation.reconcileHostPreparation(id); expect(status.items).toEqual(expect.arrayContaining([ expect.objectContaining({ label: 'Black Ops I (T5)', state: 'blocked' }), expect.objectContaining({ label: 'Black Ops II (T6)', state: 'preparing' }) ])); expect(operations(id)).toHaveLength(1); expect(operations(id)[0].payload.libraryVersionId).toBe(small.id); expect(libraries.getHostLibraryCache(id, oversized.id)).toBeNull(); }); it('does not count partial bytes from a different manifest toward download storage', async () => { const version = await publish(); const id = enroll({ runtimeImages: [runtime] }); libraries.updateHostLibraryCaches(id, [{ libraryVersionId: version.id, status: 'failed', bytesDownloaded: 80, manifestSha256: '0'.repeat(64) }]); heartbeat(id, { runtimeImages: [runtime], storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20 }); expect(preparation.reconcileHostPreparation(id).state).toBe('blocked'); expect(() => libraries.queueLibraryDownload(id, version.id)).toThrow(/insufficient storage/); expect(operations(id)).toEqual([]); }); it('resumes a failed verified-manifest download using only its remaining storage', async () => { const version = await publish(); const id = enroll({ runtimeImages: [runtime] }); preparation.reconcileHostPreparation(id); const operation = operations(id)[0]; libraries.updateHostLibraryCaches(id, [{ libraryVersionId: version.id, status: 'downloading', bytesDownloaded: 80, manifestSha256: version.manifestSha256 }]); vi.setSystemTime(Date.now() + 1); hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Connection reset' }); const failedAt = Date.now(); const free = hosts.getHost(id)!.minimumFreeBytes + version.sizeBytes - 80; heartbeat(id, { runtimeImages: [runtime], storageAvailable: free }); preparation.reconcileHostPreparation(id); expect(operations(id)).toHaveLength(1); vi.setSystemTime(failedAt + 60_000); heartbeat(id, { runtimeImages: [runtime], storageAvailable: free }); expect(preparation.reconcileHostPreparation(id).state).toBe('preparing'); expect(operations(id)).toHaveLength(2); expect(operations(id)[0].payload.libraryVersionId).toBe(version.id); expect(libraries.getHostLibraryCache(id, version.id)?.bytesDownloaded).toBe(80); }); });