import Database from 'better-sqlite3'; import { createHash } from 'node:crypto'; import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const state = vi.hoisted(() => ({ handle: null as Database.Database | null })); vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle })); import { heartbeat, createEnrollment, createHost, enrollHost, getHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js'; import { cleanupControllerLibraries, reconcileHostLibraryCleanup, createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, latestPublishedLibraries, libraryOverview, listLibraries, newestReadyLibrary, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js'; import { serveGameFile } from '../../src/lib/server/game-files/http.js'; import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js'; let directory: string; let hostId: string; let credential: string; const data = 'real game bytes'; const file = { path: 'zone/game.ff', size: Buffer.byteLength(data), sha256: createHash('sha256').update(data).digest('hex') }; beforeEach(async () => { state.handle = new Database(':memory:'); state.handle.pragma('foreign_keys=ON'); directory = await mkdtemp(join(tmpdir(), 'ec-libraries-')); vi.stubEnv('GAME_FILES_ROOT', directory); const host = createHost({ name: 'Debian host' }); hostId = host.id; const enrollment = createEnrollment(host.id); credential = enrollHost(enrollment.token, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 1_000_000_000_000 }).credential; }); afterEach(async () => { state.handle?.close(); await rm(directory, { recursive: true, force: true }); vi.unstubAllEnvs(); }); function begin() { return createLibraryImport({ hostId, game: 't6' }); } async function published() { const version = begin(); setImportManifest(hostId, version.id, [file]); await uploadImportFile(hostId, version.id, file.path, new Response(data).body); return publishImport(hostId, version.id); } function legacyLibrary(id: string, game: 't5' | 't6') { state.handle!.prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)') .run(id, `User named ${id}`, game, 'Original description', '2025-01-01T00:00:00.000Z'); } function legacyVersion(id: string, libraryId: string, createdAt: string, publishedAt: string) { state.handle!.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,published_at,storage_location,source_host_id,source_volume,manifest_sha256) VALUES(?,?,?,'Legacy version','[]','ready',?,?,?,'','','checksum')`) .run(id, libraryId, `User label ${id}`, createdAt, publishedAt, join(directory, id)); } describe('fixed game file slots', () => { it('always lists exactly T5 and T6, including before any files are uploaded', () => { expect(listLibraries()).toEqual([ { id: 't5', game: 't5', name: 'Black Ops I (T5)', description: '', createdAt: '', latestVersionId: null, versions: [] }, { id: 't6', game: 't6', name: 'Black Ops II (T6)', description: '', createdAt: '', latestVersionId: null, versions: [] } ]); expect(latestPublishedLibraries()).toEqual([]); expect(newestReadyLibrary('t5')).toBeNull(); expect(newestReadyLibrary('t6')).toBeNull(); }); it('unifies old named libraries by game without rewriting published versions or their host references', () => { listLibraries(); legacyLibrary('first-t6-library', 't6'); legacyLibrary('other-t6-library', 't6'); legacyLibrary('t5-library', 't5'); legacyVersion('old-t6', 'first-t6-library', '2025-01-03', '2025-01-04'); legacyVersion('new-t6', 'other-t6-library', '2025-01-01', '2025-01-05'); legacyVersion('t5-files', 't5-library', '2025-01-01', '2025-01-02'); updateHostLibraryCaches(hostId, [ { libraryVersionId: 'old-t6', status: 'ready', manifestSha256: 'checksum' }, { libraryVersionId: 'new-t6', status: 'ready', manifestSha256: 'checksum' } ]); const before = getLibraryVersion('old-t6'); const slots = listLibraries(); expect(slots.map(library => library.id)).toEqual(['t5', 't6']); expect(slots[1].versions.map(version => version.id)).toEqual(['old-t6', 'new-t6']); expect(slots.map(library => library.latestVersionId)).toEqual(['t5-files', 'new-t6']); expect(latestPublishedLibraries().map(version => version.id)).toEqual(['t5-files', 'new-t6']); expect(newestReadyLibrary('t6')?.id).toBe('new-t6'); const overview = libraryOverview().libraries[1]; expect(overview.versions.find(version => version.id === 'old-t6')?.caches[0].status).toBe('outdated'); expect(overview.versions.find(version => version.id === 'new-t6')?.caches[0].status).toBe('ready'); expect(getLibraryVersion('old-t6')).toEqual(before); expect(requireReadyLibrary(hostId, 'old-t6')).toMatchObject({ libraryId: 'first-t6-library', version: 'User label old-t6', storageLocation: join(directory, 'old-t6') }); expect(begin().libraryId).toBe('first-t6-library'); expect(listLibraries()[1].latestVersionId).toBe('new-t6'); }); it('breaks publication-time ties by creation time and then insertion order across old libraries', () => { listLibraries(); legacyLibrary('first-library', 't6'); legacyLibrary('second-library', 't6'); legacyVersion('first', 'first-library', '2025-01-01', '2025-01-05'); legacyVersion('second', 'second-library', '2025-01-02', '2025-01-05'); legacyVersion('third', 'first-library', '2025-01-02', '2025-01-05'); legacyVersion('later-created', 'second-library', '2025-01-04', '2025-01-04'); expect(newestReadyLibrary('t6')?.id).toBe('third'); expect(latestPublishedLibraries().map(version => version.id)).toEqual(['third']); expect(listLibraries()[1].latestVersionId).toBe('third'); }); it('ignores user-supplied library metadata when importing from hosts', () => { const first = createLibraryImport({ hostId, game: 't6', name: 'shared', version: '1', description: 'Old label' }); const next = createLibraryImport({ hostId, game: 't6', name: 'shared', version: '1', description: 'Old label' }); expect(first.description).toBe(''); expect(first.version).not.toBe('1'); expect(next.version).not.toBe(first.version); expect(first.libraryId).toBe(next.libraryId); }); }); describe('game-file import lifecycle', () => { it('publishes verified filesystem content and permanently pins manifest metadata', async () => { const initial = begin(); expect(getHostOperation(initial.operationId!)?.payload.sourceVolume).toBe('t6_game_files'); setImportManifest(hostId, initial.id, [file]); await expect(publishImport(hostId, initial.id)).rejects.toThrow(); expect(getLibraryVersion(initial.id)?.state).toBe('importing'); await uploadImportFile(hostId, initial.id, file.path, new Response(data).body); const ready = await publishImport(hostId, initial.id); expect(ready.state).toBe('ready'); expect(ready.sizeBytes).toBe(data.length); expect(await readFile(join(ready.storageLocation, file.path), 'utf8')).toBe(data); expect((await publishImport(hostId, ready.id)).id).toBe(ready.id); await expect(uploadImportFile(hostId, ready.id, file.path, new Response(data).body)).rejects.toThrow(/not writable/); expect(() => state.handle!.prepare('UPDATE game_file_versions SET description=? WHERE id=?').run('changed', ready.id)).toThrow(/immutable/); const next = begin(); expect(next.id).not.toBe(ready.id); expect(next.version).not.toBe(ready.version); expect(next.libraryId).toBe(ready.libraryId); }); it('restricts an import to its source host and rejects manifest mutation', () => { const version = begin(); expect(() => setImportManifest('another-host', version.id, [file])).toThrow(/not found/); const first = setImportManifest(hostId, version.id, [file]); expect(setImportManifest(hostId, version.id, [file]).manifestSha256).toBe(first.manifestSha256); expect(() => setImportManifest(hostId, version.id, [{ ...file, path: 'changed.ff' }])).toThrow(/cannot change/); }); it('surfaces durable agent failures while preserving historical versions', () => { const version = begin(); reportHostOperation(hostId, version.operationId!, { state: 'failed', error: 'Source volume not found' }); expect(listLibraries().find(library => library.game === 't6')?.versions[0]).toMatchObject({ state: 'failed', error: 'Source volume not found' }); }); it('returns a pollable publication status while large files are verified asynchronously', async () => { const version = begin(); setImportManifest(hostId, version.id, [file]); await uploadImportFile(hostId, version.id, file.path, new Response(data).body); expect(importPublicationStatus(hostId, version.id).status).toBe('pending'); expect(requestImportPublication(hostId, version.id).status).toBe('verifying'); expect(requestImportPublication(hostId, version.id).status).toBe('verifying'); await publishImport(hostId, version.id); expect(importPublicationStatus(hostId, version.id)).toMatchObject({ status: 'published', version: { id: version.id, manifestSha256: expect.any(String) } }); }); }); describe('verified host distribution', () => { it('deduplicates active downloads and blocks readiness without exact manifest verification', async () => { const version = await published(); const operation = queueLibraryDownload(hostId, version.id); expect(queueLibraryDownload(hostId, version.id).id).toBe(operation.id); expect(() => requireReadyLibrary(hostId, version.id)).toThrow(/finish downloading/); updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'wrong' }]); expect(getHostLibraryCache(hostId, version.id)?.status).toBe('failed'); updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]); expect(requireReadyLibrary(hostId, version.id).id).toBe(version.id); updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'outdated', manifestSha256: version.manifestSha256 }]); expect(requireReadyLibrary(hostId, version.id).id).toBe(version.id); updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', bytesDownloaded: 1, manifestSha256: version.manifestSha256 }]); expect(getHostLibraryCache(hostId, version.id)?.status).toBe('failed'); updateHostLibraryCaches(hostId, [], { snapshot: true }); expect(getHostLibraryCache(hostId, version.id)?.status).toBe('missing'); expect(() => requireReadyLibrary(hostId, version.id)).toThrow(/finish downloading/); }); it('streams resumable byte ranges and guards the immutable manifest endpoint with revocable host credentials', async () => { const version = await published(); const request = (headers: HeadersInit = {}) => new Request('https://controller/api/agent/libraries/' + version.id + '/manifest', { headers }); expect(manifestRoute({ request: request(), params: { id: version.id } }).status).toBe(401); const authorized = request({ Authorization: 'Bearer ' + credential }); expect(manifestRoute({ request: authorized, params: { id: version.id } }).status).toBe(200); const partial = await serveGameFile(request({ Range: 'bytes=5-' }), version.id, file.path); expect(partial.status).toBe(206); expect(partial.headers.get('content-range')).toBe(`bytes 5-${data.length - 1}/${data.length}`); expect(await partial.text()).toBe(data.slice(5)); const invalid = await serveGameFile(request({ Range: 'bytes=999-' }), version.id, file.path); expect(invalid.status).toBe(416); const resumed = await serveGameFile(request({ Range: 'bytes=5-', 'If-Range': '"different"' }), version.id, file.path); expect(resumed.status).toBe(200); expect(await resumed.text()).toBe(data); revokeCredentials(hostId); expect(manifestRoute({ request: authorized, params: { id: version.id } }).status).toBe(401); }); }); describe('obsolete game-file cleanup', () => { beforeEach(() => { state.handle!.exec('CREATE TABLE native_instances (id TEXT, host_id TEXT, library_id TEXT, retired_at TEXT)'); }); async function obsolete() { const old = await published(); reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} }); const latest = await published(); return { old, latest }; } it('retains history and the latest files while removing unreferenced old bytes', async () => { const { old, latest } = await obsolete(); await cleanupControllerLibraries(); expect(getLibraryVersion(old.id)).toMatchObject({ state: 'ready', filesRemoved: true, files: [file] }); await expect(readFile(join(old.storageLocation, file.path))).rejects.toThrow(); expect(await readFile(join(latest.storageLocation, file.path), 'utf8')).toBe(data); expect(() => queueLibraryDownload(hostId, old.id)).toThrow('removed'); expect((await serveGameFile(new Request('http://local/file'), old.id, file.path)).status).toBe(410); await cleanupControllerLibraries(); expect(listLibraries()[1].versions).toHaveLength(2); }); it('resumes a reserved controller deletion after restart and preserves the manifest', async () => { const old = await published(); reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} }); updateHostLibraryCaches(hostId, [{ id: old.id, status: 'ready', manifestSha256: old.manifestSha256 }]); await published(); updateHostLibraryCaches(hostId, [], { snapshot: true }); state.handle!.prepare('INSERT INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(old.id, new Date().toISOString()); await rm(old.storageLocation, { recursive: true }); expect(getLibraryVersion(old.id)).toMatchObject({ filesRemoving: true, filesRemoved: false }); expect(() => queueLibraryDownload(hostId, old.id)).toThrow('removed'); await cleanupControllerLibraries(); expect(getLibraryVersion(old.id)).toMatchObject({ filesRemoving: false, filesRemoved: true, files: [file] }); }); it('does not send cleanup to agents without the capability or erase their cached files', async () => { const old = await published(); reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} }); updateHostLibraryCaches(hostId, [{ id: old.id, status: 'ready', manifestSha256: old.manifestSha256 }]); await published(); reconcileHostLibraryCleanup(hostId); expect(state.handle!.prepare("SELECT * FROM ec_host_operations WHERE kind='library.remove'").all()).toHaveLength(0); expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false); }); it('pins files for stopped or provisioning instances until retirement', async () => { const old = await published(); reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} }); state.handle!.prepare('INSERT INTO native_instances VALUES(?,?,?,NULL)').run('instance', hostId, old.id); await published(); await cleanupControllerLibraries(); expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false); state.handle!.exec("UPDATE native_instances SET retired_at='now'"); await cleanupControllerLibraries(); expect(getLibraryVersion(old.id)?.filesRemoved).toBe(true); }); it('waits for offline hosts, partial copies, and unresolved downloads', async () => { const old = await published(); reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} }); updateHostLibraryCaches(hostId, [{ id: old.id, status: 'missing', bytesDownloaded: 1 }]); await published(); await cleanupControllerLibraries(); expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false); const operation = queueLibraryDownload(hostId, old.id); updateHostLibraryCaches(hostId, [], { snapshot: true }); await cleanupControllerLibraries(); expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false); reportHostOperation(hostId, operation.id, { state: 'failed', error: 'interrupted' }); await cleanupControllerLibraries(); expect(getLibraryVersion(old.id)?.filesRemoved).toBe(true); }); it('queues host removal only without instance references and waits for absence before controller cleanup', async () => { heartbeat(hostId, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, capabilities: ['library.remove'], storageAvailable: 1_000_000_000_000 }); const old = await published(); reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} }); updateHostLibraryCaches(hostId, [{ id: old.id, status: 'ready', manifestSha256: old.manifestSha256 }]); state.handle!.prepare('INSERT INTO native_instances VALUES(?,?,?,NULL)').run('instance', hostId, old.id); await published(); reconcileHostLibraryCleanup(hostId); expect(state.handle!.prepare("SELECT * FROM ec_host_operations WHERE kind='library.remove'").all()).toHaveLength(0); state.handle!.exec("UPDATE native_instances SET retired_at='now'"); reconcileHostLibraryCleanup(hostId); reconcileHostLibraryCleanup(hostId); const removals = state.handle!.prepare("SELECT id FROM ec_host_operations WHERE kind='library.remove'").all() as { id: string }[]; expect(removals).toHaveLength(1); expect(() => queueLibraryDownload(hostId, old.id)).toThrow('being removed'); reportHostOperation(hostId, removals[0].id, { state: 'completed', result: { removed: true } }); await cleanupControllerLibraries(); expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false); updateHostLibraryCaches(hostId, [], { snapshot: true }); await cleanupControllerLibraries(); expect(getLibraryVersion(old.id)?.filesRemoved).toBe(true); }); });