276 lines
19 KiB
TypeScript
276 lines
19 KiB
TypeScript
import Database from 'better-sqlite3';
|
|
import { createHash } from 'node:crypto';
|
|
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
const state = vi.hoisted(() => ({ handle: null as Database.Database | null }));
|
|
vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle }));
|
|
import { heartbeat, createEnrollment, createHost, enrollHost, getHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js';
|
|
import { cleanupControllerLibraries, reconcileHostLibraryCleanup, createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, latestPublishedLibraries, libraryOverview, listLibraries, newestReadyLibrary, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js';
|
|
import { serveGameFile } from '../../src/lib/server/game-files/http.js';
|
|
import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js';
|
|
|
|
let directory: string; let hostId: string; let credential: string;
|
|
const data = 'real game bytes';
|
|
const file = { path: 'zone/game.ff', size: Buffer.byteLength(data), sha256: createHash('sha256').update(data).digest('hex') };
|
|
beforeEach(async () => {
|
|
state.handle = new Database(':memory:'); state.handle.pragma('foreign_keys=ON');
|
|
directory = await mkdtemp(join(tmpdir(), 'ec-libraries-')); vi.stubEnv('GAME_FILES_ROOT', directory);
|
|
const host = createHost({ name: 'Debian host' }); hostId = host.id;
|
|
const enrollment = createEnrollment(host.id);
|
|
credential = enrollHost(enrollment.token, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 1_000_000_000_000 }).credential;
|
|
});
|
|
afterEach(async () => { state.handle?.close(); await rm(directory, { recursive: true, force: true }); vi.unstubAllEnvs(); });
|
|
function begin() { return createLibraryImport({ hostId, game: 't6' }); }
|
|
async function published() {
|
|
const version = begin(); setImportManifest(hostId, version.id, [file]);
|
|
await uploadImportFile(hostId, version.id, file.path, new Response(data).body);
|
|
return publishImport(hostId, version.id);
|
|
}
|
|
|
|
function legacyLibrary(id: string, game: 't5' | 't6') {
|
|
state.handle!.prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)')
|
|
.run(id, `User named ${id}`, game, 'Original description', '2025-01-01T00:00:00.000Z');
|
|
}
|
|
function legacyVersion(id: string, libraryId: string, createdAt: string, publishedAt: string) {
|
|
state.handle!.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,published_at,storage_location,source_host_id,source_volume,manifest_sha256)
|
|
VALUES(?,?,?,'Legacy version','[]','ready',?,?,?,'','','checksum')`)
|
|
.run(id, libraryId, `User label ${id}`, createdAt, publishedAt, join(directory, id));
|
|
}
|
|
|
|
describe('fixed game file slots', () => {
|
|
it('always lists exactly T5 and T6, including before any files are uploaded', () => {
|
|
expect(listLibraries()).toEqual([
|
|
{ id: 't5', game: 't5', name: 'Black Ops I (T5)', description: '', createdAt: '', latestVersionId: null, versions: [] },
|
|
{ id: 't6', game: 't6', name: 'Black Ops II (T6)', description: '', createdAt: '', latestVersionId: null, versions: [] }
|
|
]);
|
|
expect(latestPublishedLibraries()).toEqual([]);
|
|
expect(newestReadyLibrary('t5')).toBeNull();
|
|
expect(newestReadyLibrary('t6')).toBeNull();
|
|
});
|
|
|
|
it('unifies old named libraries by game without rewriting published versions or their host references', () => {
|
|
listLibraries();
|
|
legacyLibrary('first-t6-library', 't6');
|
|
legacyLibrary('other-t6-library', 't6');
|
|
legacyLibrary('t5-library', 't5');
|
|
legacyVersion('old-t6', 'first-t6-library', '2025-01-03', '2025-01-04');
|
|
legacyVersion('new-t6', 'other-t6-library', '2025-01-01', '2025-01-05');
|
|
legacyVersion('t5-files', 't5-library', '2025-01-01', '2025-01-02');
|
|
updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: 'old-t6', status: 'ready', manifestSha256: 'checksum' },
|
|
{ libraryVersionId: 'new-t6', status: 'ready', manifestSha256: 'checksum' }
|
|
]);
|
|
const before = getLibraryVersion('old-t6');
|
|
const slots = listLibraries();
|
|
expect(slots.map(library => library.id)).toEqual(['t5', 't6']);
|
|
expect(slots[1].versions.map(version => version.id)).toEqual(['old-t6', 'new-t6']);
|
|
expect(slots.map(library => library.latestVersionId)).toEqual(['t5-files', 'new-t6']);
|
|
expect(latestPublishedLibraries().map(version => version.id)).toEqual(['t5-files', 'new-t6']);
|
|
expect(newestReadyLibrary('t6')?.id).toBe('new-t6');
|
|
const overview = libraryOverview().libraries[1];
|
|
expect(overview.versions.find(version => version.id === 'old-t6')?.caches[0].status).toBe('outdated');
|
|
expect(overview.versions.find(version => version.id === 'new-t6')?.caches[0].status).toBe('ready');
|
|
expect(getLibraryVersion('old-t6')).toEqual(before);
|
|
expect(requireReadyLibrary(hostId, 'old-t6')).toMatchObject({
|
|
libraryId: 'first-t6-library', version: 'User label old-t6', storageLocation: join(directory, 'old-t6')
|
|
});
|
|
expect(begin().libraryId).toBe('first-t6-library');
|
|
expect(listLibraries()[1].latestVersionId).toBe('new-t6');
|
|
});
|
|
|
|
it('breaks publication-time ties by creation time and then insertion order across old libraries', () => {
|
|
listLibraries();
|
|
legacyLibrary('first-library', 't6');
|
|
legacyLibrary('second-library', 't6');
|
|
legacyVersion('first', 'first-library', '2025-01-01', '2025-01-05');
|
|
legacyVersion('second', 'second-library', '2025-01-02', '2025-01-05');
|
|
legacyVersion('third', 'first-library', '2025-01-02', '2025-01-05');
|
|
legacyVersion('later-created', 'second-library', '2025-01-04', '2025-01-04');
|
|
expect(newestReadyLibrary('t6')?.id).toBe('third');
|
|
expect(latestPublishedLibraries().map(version => version.id)).toEqual(['third']);
|
|
expect(listLibraries()[1].latestVersionId).toBe('third');
|
|
});
|
|
|
|
it('ignores user-supplied library metadata when importing from hosts', () => {
|
|
const first = createLibraryImport({ hostId, game: 't6', name: 'shared', version: '1', description: 'Old label' });
|
|
const next = createLibraryImport({ hostId, game: 't6', name: 'shared', version: '1', description: 'Old label' });
|
|
expect(first.description).toBe('');
|
|
expect(first.version).not.toBe('1');
|
|
expect(next.version).not.toBe(first.version);
|
|
expect(first.libraryId).toBe(next.libraryId);
|
|
});
|
|
});
|
|
|
|
describe('game-file import lifecycle', () => {
|
|
it('publishes verified filesystem content and permanently pins manifest metadata', async () => {
|
|
const initial = begin();
|
|
expect(getHostOperation(initial.operationId!)?.payload.sourceVolume).toBe('t6_game_files');
|
|
setImportManifest(hostId, initial.id, [file]);
|
|
await expect(publishImport(hostId, initial.id)).rejects.toThrow();
|
|
expect(getLibraryVersion(initial.id)?.state).toBe('importing');
|
|
await uploadImportFile(hostId, initial.id, file.path, new Response(data).body);
|
|
const ready = await publishImport(hostId, initial.id);
|
|
expect(ready.state).toBe('ready'); expect(ready.sizeBytes).toBe(data.length);
|
|
expect(await readFile(join(ready.storageLocation, file.path), 'utf8')).toBe(data);
|
|
expect((await publishImport(hostId, ready.id)).id).toBe(ready.id);
|
|
await expect(uploadImportFile(hostId, ready.id, file.path, new Response(data).body)).rejects.toThrow(/not writable/);
|
|
expect(() => state.handle!.prepare('UPDATE game_file_versions SET description=? WHERE id=?').run('changed', ready.id)).toThrow(/immutable/);
|
|
const next = begin();
|
|
expect(next.id).not.toBe(ready.id);
|
|
expect(next.version).not.toBe(ready.version);
|
|
expect(next.libraryId).toBe(ready.libraryId);
|
|
});
|
|
it('restricts an import to its source host and rejects manifest mutation', () => {
|
|
const version = begin();
|
|
expect(() => setImportManifest('another-host', version.id, [file])).toThrow(/not found/);
|
|
const first = setImportManifest(hostId, version.id, [file]);
|
|
expect(setImportManifest(hostId, version.id, [file]).manifestSha256).toBe(first.manifestSha256);
|
|
expect(() => setImportManifest(hostId, version.id, [{ ...file, path: 'changed.ff' }])).toThrow(/cannot change/);
|
|
});
|
|
it('surfaces durable agent failures while preserving historical versions', () => {
|
|
const version = begin();
|
|
reportHostOperation(hostId, version.operationId!, { state: 'failed', error: 'Source volume not found' });
|
|
expect(listLibraries().find(library => library.game === 't6')?.versions[0]).toMatchObject({ state: 'failed', error: 'Source volume not found' });
|
|
});
|
|
it('returns a pollable publication status while large files are verified asynchronously', async () => {
|
|
const version = begin(); setImportManifest(hostId, version.id, [file]);
|
|
await uploadImportFile(hostId, version.id, file.path, new Response(data).body);
|
|
expect(importPublicationStatus(hostId, version.id).status).toBe('pending');
|
|
expect(requestImportPublication(hostId, version.id).status).toBe('verifying');
|
|
expect(requestImportPublication(hostId, version.id).status).toBe('verifying');
|
|
await publishImport(hostId, version.id);
|
|
expect(importPublicationStatus(hostId, version.id)).toMatchObject({ status: 'published', version: { id: version.id, manifestSha256: expect.any(String) } });
|
|
});
|
|
});
|
|
describe('verified host distribution', () => {
|
|
it('deduplicates active downloads and blocks readiness without exact manifest verification', async () => {
|
|
const version = await published();
|
|
const operation = queueLibraryDownload(hostId, version.id);
|
|
expect(queueLibraryDownload(hostId, version.id).id).toBe(operation.id);
|
|
expect(() => requireReadyLibrary(hostId, version.id)).toThrow(/finish downloading/);
|
|
updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'wrong' }]);
|
|
expect(getHostLibraryCache(hostId, version.id)?.status).toBe('failed');
|
|
updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]);
|
|
expect(requireReadyLibrary(hostId, version.id).id).toBe(version.id);
|
|
updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'outdated', manifestSha256: version.manifestSha256 }]);
|
|
expect(requireReadyLibrary(hostId, version.id).id).toBe(version.id);
|
|
updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', bytesDownloaded: 1, manifestSha256: version.manifestSha256 }]);
|
|
expect(getHostLibraryCache(hostId, version.id)?.status).toBe('failed');
|
|
updateHostLibraryCaches(hostId, [], { snapshot: true });
|
|
expect(getHostLibraryCache(hostId, version.id)?.status).toBe('missing');
|
|
expect(() => requireReadyLibrary(hostId, version.id)).toThrow(/finish downloading/);
|
|
});
|
|
it('streams resumable byte ranges and guards the immutable manifest endpoint with revocable host credentials', async () => {
|
|
const version = await published();
|
|
const request = (headers: HeadersInit = {}) => new Request('https://controller/api/agent/libraries/' + version.id + '/manifest', { headers });
|
|
expect(manifestRoute({ request: request(), params: { id: version.id } }).status).toBe(401);
|
|
const authorized = request({ Authorization: 'Bearer ' + credential });
|
|
expect(manifestRoute({ request: authorized, params: { id: version.id } }).status).toBe(200);
|
|
const partial = await serveGameFile(request({ Range: 'bytes=5-' }), version.id, file.path);
|
|
expect(partial.status).toBe(206); expect(partial.headers.get('content-range')).toBe(`bytes 5-${data.length - 1}/${data.length}`);
|
|
expect(await partial.text()).toBe(data.slice(5));
|
|
const invalid = await serveGameFile(request({ Range: 'bytes=999-' }), version.id, file.path);
|
|
expect(invalid.status).toBe(416);
|
|
const resumed = await serveGameFile(request({ Range: 'bytes=5-', 'If-Range': '"different"' }), version.id, file.path);
|
|
expect(resumed.status).toBe(200); expect(await resumed.text()).toBe(data);
|
|
revokeCredentials(hostId);
|
|
expect(manifestRoute({ request: authorized, params: { id: version.id } }).status).toBe(401);
|
|
});
|
|
});
|
|
|
|
|
|
describe('obsolete game-file cleanup', () => {
|
|
beforeEach(() => {
|
|
state.handle!.exec('CREATE TABLE native_instances (id TEXT, host_id TEXT, library_id TEXT, retired_at TEXT)');
|
|
});
|
|
async function obsolete() {
|
|
const old = await published();
|
|
reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
|
|
const latest = await published();
|
|
return { old, latest };
|
|
}
|
|
it('retains history and the latest files while removing unreferenced old bytes', async () => {
|
|
const { old, latest } = await obsolete();
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)).toMatchObject({ state: 'ready', filesRemoved: true, files: [file] });
|
|
await expect(readFile(join(old.storageLocation, file.path))).rejects.toThrow();
|
|
expect(await readFile(join(latest.storageLocation, file.path), 'utf8')).toBe(data);
|
|
expect(() => queueLibraryDownload(hostId, old.id)).toThrow('removed');
|
|
expect((await serveGameFile(new Request('http://local/file'), old.id, file.path)).status).toBe(410);
|
|
await cleanupControllerLibraries();
|
|
expect(listLibraries()[1].versions).toHaveLength(2);
|
|
});
|
|
it('resumes a reserved controller deletion after restart and preserves the manifest', async () => {
|
|
const old = await published();
|
|
reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
|
|
updateHostLibraryCaches(hostId, [{ id: old.id, status: 'ready', manifestSha256: old.manifestSha256 }]);
|
|
await published();
|
|
updateHostLibraryCaches(hostId, [], { snapshot: true });
|
|
state.handle!.prepare('INSERT INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(old.id, new Date().toISOString());
|
|
await rm(old.storageLocation, { recursive: true });
|
|
expect(getLibraryVersion(old.id)).toMatchObject({ filesRemoving: true, filesRemoved: false });
|
|
expect(() => queueLibraryDownload(hostId, old.id)).toThrow('removed');
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)).toMatchObject({ filesRemoving: false, filesRemoved: true, files: [file] });
|
|
});
|
|
it('does not send cleanup to agents without the capability or erase their cached files', async () => {
|
|
const old = await published();
|
|
reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
|
|
updateHostLibraryCaches(hostId, [{ id: old.id, status: 'ready', manifestSha256: old.manifestSha256 }]);
|
|
await published();
|
|
reconcileHostLibraryCleanup(hostId);
|
|
expect(state.handle!.prepare("SELECT * FROM ec_host_operations WHERE kind='library.remove'").all()).toHaveLength(0);
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
|
|
});
|
|
it('pins files for stopped or provisioning instances until retirement', async () => {
|
|
const old = await published();
|
|
reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
|
|
state.handle!.prepare('INSERT INTO native_instances VALUES(?,?,?,NULL)').run('instance', hostId, old.id);
|
|
await published();
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
|
|
state.handle!.exec("UPDATE native_instances SET retired_at='now'");
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(true);
|
|
});
|
|
it('waits for offline hosts, partial copies, and unresolved downloads', async () => {
|
|
const old = await published();
|
|
reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
|
|
updateHostLibraryCaches(hostId, [{ id: old.id, status: 'missing', bytesDownloaded: 1 }]);
|
|
await published();
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
|
|
const operation = queueLibraryDownload(hostId, old.id);
|
|
updateHostLibraryCaches(hostId, [], { snapshot: true });
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
|
|
reportHostOperation(hostId, operation.id, { state: 'failed', error: 'interrupted' });
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(true);
|
|
});
|
|
it('queues host removal only without instance references and waits for absence before controller cleanup', async () => {
|
|
heartbeat(hostId, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, capabilities: ['library.remove'], storageAvailable: 1_000_000_000_000 });
|
|
const old = await published();
|
|
reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
|
|
updateHostLibraryCaches(hostId, [{ id: old.id, status: 'ready', manifestSha256: old.manifestSha256 }]);
|
|
state.handle!.prepare('INSERT INTO native_instances VALUES(?,?,?,NULL)').run('instance', hostId, old.id);
|
|
await published();
|
|
reconcileHostLibraryCleanup(hostId);
|
|
expect(state.handle!.prepare("SELECT * FROM ec_host_operations WHERE kind='library.remove'").all()).toHaveLength(0);
|
|
state.handle!.exec("UPDATE native_instances SET retired_at='now'");
|
|
reconcileHostLibraryCleanup(hostId);
|
|
reconcileHostLibraryCleanup(hostId);
|
|
const removals = state.handle!.prepare("SELECT id FROM ec_host_operations WHERE kind='library.remove'").all() as { id: string }[];
|
|
expect(removals).toHaveLength(1);
|
|
expect(() => queueLibraryDownload(hostId, old.id)).toThrow('being removed');
|
|
reportHostOperation(hostId, removals[0].id, { state: 'completed', result: { removed: true } });
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
|
|
updateHostLibraryCaches(hostId, [], { snapshot: true });
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(true);
|
|
});
|
|
});
|