From c8c53e87d6b609f7a88409955b006cdce06706fa Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 5 Oct 2026 15:51:55 -0400 Subject: [PATCH] Lighten library cache reports and heartbeat cleanup --- src/lib/server/game-files/index.ts | 13 +++- src/routes/api/agent/heartbeat/+server.ts | 3 +- tests/unit/game-file-library.test.ts | 73 +++++++++++++++++++++++ tests/unit/host-preparation.test.ts | 17 ++++++ tests/unit/worker-scheduling.test.ts | 24 ++++++++ 5 files changed, 127 insertions(+), 3 deletions(-) diff --git a/src/lib/server/game-files/index.ts b/src/lib/server/game-files/index.ts index 4cc60ab..e40c7b7 100644 --- a/src/lib/server/game-files/index.ts +++ b/src/lib/server/game-files/index.ts @@ -332,9 +332,17 @@ function transferProgress(value: unknown, status: LibraryCacheState): LibraryTra bytesPerSecond: progress.bytesPerSecond as number | null ?? null, etaSeconds: progress.etaSeconds as number | null ?? null }; } +type LibraryCacheMetadata = Pick; +const publishedCacheMetadata = new WeakMap, Map>(); + export function updateHostLibraryCaches(hostId: string, input: unknown, options: { snapshot?: boolean } = {}): void { if (!Array.isArray(input) || input.length > 2000) throw new Error('Invalid host library report.'); const handle = database(); + // Published metadata is immutable. Keep caches local to the connection and + // avoid retaining a publication from an outer transaction that may roll back. + const versions = handle.inTransaction ? new Map() + : publishedCacheMetadata.get(handle) ?? new Map(); + if (!handle.inTransaction) publishedCacheMetadata.set(handle, versions); const now = new Date().toISOString(); handle.transaction(() => { const reported = new Set(); @@ -344,7 +352,10 @@ export function updateHostLibraryCaches(hostId: string, input: unknown, options: if (typeof id !== 'string') throw new Error('Invalid library version ID.'); if (reported.has(id)) throw new Error('Duplicate version in host cache report.'); reported.add(id); - const version = getLibraryVersion(id); if (!version || version.state !== 'ready') continue; + const version = versions.get(id) ?? handle.prepare(`SELECT state, size_bytes AS sizeBytes, manifest_sha256 AS manifestSha256 + FROM game_file_versions WHERE id=?`).get(id) as LibraryCacheMetadata | undefined; + if (!version || version.state !== 'ready') continue; + versions.set(id, version); let status = raw.status ?? raw.state; if (!['missing', 'downloading', 'verifying', 'ready', 'failed', 'outdated'].includes(status)) throw new Error('Invalid library cache state.'); const checksum = typeof raw.manifestSha256 === 'string' ? raw.manifestSha256 : null; diff --git a/src/routes/api/agent/heartbeat/+server.ts b/src/routes/api/agent/heartbeat/+server.ts index 40e56b4..dc5773b 100644 --- a/src/routes/api/agent/heartbeat/+server.ts +++ b/src/routes/api/agent/heartbeat/+server.ts @@ -2,7 +2,7 @@ import { json } from '@sveltejs/kit'; import { heartbeat, pendingCreationCancellations } from '$lib/server/hosts/index.js'; import { authenticateAgent, hostApiError, objectBody } from '$lib/server/hosts/http.js'; import { reconcileHostPreparation } from '$lib/server/hosts/preparation.js'; -import { reconcileHostLibraryCleanup, cleanupControllerLibraries } from '$lib/server/game-files/index.js'; +import { reconcileHostLibraryCleanup } from '$lib/server/game-files/index.js'; import { fleetSettings } from '$lib/server/settings.js'; export async function POST({ request }: { request: Request }) { const authenticated = authenticateAgent(request); @@ -16,7 +16,6 @@ export async function POST({ request }: { request: Request }) { } reconcileHostPreparation(host.id); reconcileHostLibraryCleanup(host.id); - await cleanupControllerLibraries(); return json({ host, cancelledCreations: pendingCreationCancellations(host.id), heartbeatSeconds: 15, observationKeepDays: fleetSettings().observationKeepDays, serverTime: new Date().toISOString() }); } catch (cause) { return hostApiError(cause); } diff --git a/tests/unit/game-file-library.test.ts b/tests/unit/game-file-library.test.ts index 1c79f45..67a9342 100644 --- a/tests/unit/game-file-library.test.ts +++ b/tests/unit/game-file-library.test.ts @@ -145,6 +145,79 @@ describe('game-file import lifecycle', () => { }); }); describe('verified host distribution', () => { + it('reuses published metadata across host reports without reading or parsing a large manifest', () => { + const version = begin(); + const manifest = JSON.stringify(Array.from({ length: 20_000 }, (_, i) => ({ ...file, path: `zone/file-${i}.ff` }))); + const size = file.size * 20_000; + state.handle!.prepare("UPDATE game_file_versions SET state='ready',size_bytes=?,manifest=?,manifest_sha256='large-checksum' WHERE id=?") + .run(size, manifest, version.id); + const otherHost = createHost({ name: 'Other reporting host' }); + const parse = JSON.parse; + const parsing = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => { + if (value === manifest) throw new Error('Cache reports must not deserialize manifests.'); + return parse(value, reviver); + }); + const preparing = vi.spyOn(state.handle!, 'prepare'); + try { + for (const id of [hostId, otherHost.id, hostId]) { + updateHostLibraryCaches(id, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'large-checksum' }]); + expect(getHostLibraryCache(id, version.id)).toMatchObject({ status: 'ready', bytesDownloaded: size, bytesTotal: size }); + } + const reads = preparing.mock.calls.map(([sql]) => sql).filter(sql => sql.includes('FROM game_file_versions')); + expect(reads).toHaveLength(1); + expect(state.handle!.prepare(reads[0]).columns().map(column => column.name)).toEqual(['state', 'sizeBytes', 'manifestSha256']); + updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'wrong' }]); + expect(getHostLibraryCache(hostId, version.id)?.status).toBe('failed'); + expect(() => updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'downloading', bytesDownloaded: size + 1 }])) + .toThrow('Invalid library download progress'); + } finally { parsing.mockRestore(); preparing.mockRestore(); } + }); + + it('does not cache missing or unpublished versions', () => { + updateHostLibraryCaches(hostId, [{ libraryVersionId: 'future', status: 'ready', manifestSha256: 'checksum' }]); + const version = begin(); + state.handle!.prepare("UPDATE game_file_versions SET id='future' WHERE id=?").run(version.id); + updateHostLibraryCaches(hostId, [{ libraryVersionId: 'future', status: 'ready', manifestSha256: 'checksum' }]); + expect(getHostLibraryCache(hostId, 'future')).toBeNull(); + state.handle!.prepare("UPDATE game_file_versions SET state='ready',size_bytes=123,manifest_sha256='checksum' WHERE id='future'").run(); + updateHostLibraryCaches(hostId, [{ libraryVersionId: 'future', status: 'ready', manifestSha256: 'checksum' }]); + expect(getHostLibraryCache(hostId, 'future')).toMatchObject({ status: 'ready', bytesTotal: 123 }); + }); + + it('does not retain metadata from a rolled-back publication', () => { + const version = begin(); + expect(() => state.handle!.transaction(() => { + state.handle!.prepare("UPDATE game_file_versions SET state='ready',size_bytes=123,manifest_sha256='temporary' WHERE id=?").run(version.id); + updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'temporary' }]); + throw new Error('rollback'); + })()).toThrow('rollback'); + updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'temporary' }]); + expect(getHostLibraryCache(hostId, version.id)).toBeNull(); + state.handle!.prepare("UPDATE game_file_versions SET state='ready',size_bytes=456,manifest_sha256='published' WHERE id=?").run(version.id); + updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'published' }]); + expect(getHostLibraryCache(hostId, version.id)).toMatchObject({ status: 'ready', bytesTotal: 456 }); + }); + + it('keeps metadata caches separate for each database connection', () => { + const seed = () => { + listLibraries(); + legacyLibrary('cache-library', 't6'); + legacyVersion('cache-version', 'cache-library', '2025-01-01', '2025-01-01'); + }; + const report = () => updateHostLibraryCaches(hostId, [{ libraryVersionId: 'cache-version', status: 'ready', manifestSha256: 'checksum' }]); + seed(); report(); + const original = state.handle!; + state.handle = new Database(':memory:'); + try { + seed(); + const preparing = vi.spyOn(state.handle, 'prepare'); + try { + report(); + expect(preparing.mock.calls.filter(([sql]) => sql.includes('FROM game_file_versions'))).toHaveLength(1); + } finally { preparing.mockRestore(); } + } finally { state.handle.close(); state.handle = original; } + }); + it('deduplicates active downloads and blocks readiness without exact manifest verification', async () => { const version = await published(); const operation = queueLibraryDownload(hostId, version.id); diff --git a/tests/unit/host-preparation.test.ts b/tests/unit/host-preparation.test.ts index 1f76b27..cc02006 100644 --- a/tests/unit/host-preparation.test.ts +++ b/tests/unit/host-preparation.test.ts @@ -68,6 +68,23 @@ function verified(id: string, version: GameLibraryVersion) { } describe('automatic host preparation', () => { + it('accepts heartbeats without running controller library cleanup', async () => { + const host = hosts.createHost({ name: 'Heartbeat host' }); + const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory); + const { POST } = await import('../../src/routes/api/agent/heartbeat/+server.js'); + const cleanup = vi.spyOn(libraries, 'cleanupControllerLibraries').mockRejectedValue(new Error('Cleanup is independent of heartbeats')); + try { + for (let i = 0; i < 3; i++) { + const response = await POST({ request: new Request('https://controller.test/api/agent/heartbeat', { + method: 'POST', headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ inventory: { ...inventory, libraryCaches: [] } }) + }) }); + expect(response.status).toBe(200); + } + expect(cleanup).not.toHaveBeenCalled(); + } finally { cleanup.mockRestore(); } + }); + it('keeps re-enabled hosts disabled until a fresh runtime pull and both game libraries are verified', async () => { const t5 = await publish('t5'); const t6 = await publish('t6'); diff --git a/tests/unit/worker-scheduling.test.ts b/tests/unit/worker-scheduling.test.ts index ba5ab15..6a392ce 100644 --- a/tests/unit/worker-scheduling.test.ts +++ b/tests/unit/worker-scheduling.test.ts @@ -31,6 +31,30 @@ async function setup() { } describe('independent provisioning scheduling', () => { + it('runs controller library cleanup on worker ticks without overlapping a slow cleanup', async () => { + const s = await setup(); + const { cleanupControllerLibraries } = await import('../../src/lib/server/game-files/index.js'); + const cleanup = deferred(); + vi.mocked(cleanupControllerLibraries).mockImplementationOnce(() => cleanup.promise); + vi.useFakeTimers(); + try { + s.startWorker(); + expect(cleanupControllerLibraries).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(30_000); + expect(cleanupControllerLibraries).toHaveBeenCalledTimes(1); + expect(s.process.mock.calls.length).toBeGreaterThan(1); + cleanup.resolve(); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(15_000); + expect(cleanupControllerLibraries).toHaveBeenCalledTimes(2); + } finally { + s.stopWorker(); cleanup.resolve(); s.background.resolve(); + await vi.advanceTimersByTimeAsync(0); + vi.useRealTimers(); + await vi.waitFor(() => expect(s.workerActivity()).toEqual([])); + } + }); + it('admits and checks new work while campaign, beta and maintenance observers are still waiting', async () => { const s = await setup(); try {