315 lines
20 KiB
TypeScript
315 lines
20 KiB
TypeScript
import { initializeStandaloneSettings } from '../helpers/settings.js';
|
|
import Database from 'better-sqlite3';
|
|
import { createHash } from 'node:crypto';
|
|
import { mkdir, mkdtemp, readFile, rename, rm, stat } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { dirname, join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
|
|
const state = vi.hoisted(() => ({ handle: null as Database.Database | null }));
|
|
vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle }));
|
|
import {
|
|
browserUploadStatus, createLibraryImport, createLibraryUpload, getHostLibraryCache,
|
|
getLibraryVersion, latestPublishedLibraries, listLibraries, newestReadyLibrary, publishBrowserUpload, queueLibraryDownload,
|
|
requestBrowserPublication, requireReadyLibrary, updateHostLibraryCaches, uploadBrowserFile
|
|
} from '../../src/lib/server/game-files/index.js';
|
|
import { createEnrollment, createHost, enrollHost, listHosts } from '../../src/lib/server/hosts/index.js';
|
|
import { manifestChecksum } from '../../src/lib/server/game-files/storage.js';
|
|
import { serveGameFile } from '../../src/lib/server/game-files/http.js';
|
|
import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js';
|
|
|
|
let directory: string;
|
|
const modified = 1_700_000_000_000;
|
|
const bytes = Buffer.from('verified Plutonium game content');
|
|
const sha = (value: Uint8Array) => createHash('sha256').update(value).digest('hex');
|
|
const file = (path = 'zone/base.ff', content = bytes) => ({ path, size: content.length, lastModified: modified });
|
|
const upload = (id: string, path: string, content: Buffer, offset = 0, checksum = sha(content)) =>
|
|
uploadBrowserFile(id, path, { offset, size: content.length, sha256: checksum }, new Response(new Uint8Array(content)).body);
|
|
const begin = (game: 't5' | 't6' = 't6', files = [file()]) => createLibraryUpload({ game, files });
|
|
|
|
beforeEach(async () => {
|
|
directory = await mkdtemp(join(tmpdir(), 'ec-browser-library-'));
|
|
state.handle = new Database(join(directory, 'controller.sqlite')); initializeStandaloneSettings(state.handle);
|
|
state.handle.pragma('foreign_keys=ON');
|
|
vi.stubEnv('GAME_FILES_ROOT', join(directory, 'libraries'));
|
|
});
|
|
afterEach(async () => {
|
|
state.handle?.close(); state.handle = null;
|
|
await rm(directory, { recursive: true, force: true });
|
|
vi.unstubAllEnvs();
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
function host() {
|
|
const created = createHost({ name: 'Fresh download target' });
|
|
const { credential } = enrollHost(createEnrollment(created.id).token, {
|
|
protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3
|
|
});
|
|
return { id: created.id, credential };
|
|
}
|
|
|
|
describe('browser folder uploads', () => {
|
|
it.each(['t5', 't6'] as const)('publishes %s from a browser without an enrolled host or DLC declarations', async game => {
|
|
expect(listHosts()).toEqual([]);
|
|
const initial = begin(game);
|
|
expect(initial).toMatchObject({ state: 'importing', operationId: null, sourceHostId: null, sourceVolume: null });
|
|
expect(initial).not.toHaveProperty('dlc');
|
|
expect(await browserUploadStatus(initial.id)).toMatchObject({
|
|
status: 'pending', files: [{ ...file(), receivedBytes: 0, complete: false }]
|
|
});
|
|
await upload(initial.id, file().path, bytes);
|
|
const accepted = await requestBrowserPublication(initial.id);
|
|
expect(['verifying', 'published']).toContain(accepted.status);
|
|
const ready = await publishBrowserUpload(initial.id);
|
|
expect(ready).toMatchObject({ state: 'ready', sizeBytes: bytes.length, fileCount: 1 });
|
|
expect(ready.files).toEqual([{ path: file().path, size: bytes.length, sha256: sha(bytes) }]);
|
|
expect(ready.manifestSha256).toBe(manifestChecksum(ready.files));
|
|
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
|
|
expect(await browserUploadStatus(initial.id)).toMatchObject({ status: 'published' });
|
|
expect(listLibraries().find(library => library.game === game)?.versions[0]).not.toHaveProperty('dlc');
|
|
expect(listHosts()).toEqual([]);
|
|
});
|
|
|
|
it('preserves nested relative contents and derives a canonical manifest in path order', async () => {
|
|
const selected = [file('zone/english/a.ff'), file('main/iw_00.iwd'), file('empty.txt', Buffer.alloc(0))];
|
|
const initial = begin('t6', selected);
|
|
for (const item of selected) await upload(initial.id, item.path, item.size ? bytes : Buffer.alloc(0));
|
|
const ready = await publishBrowserUpload(initial.id);
|
|
expect(ready.files.map(item => item.path)).toEqual(['empty.txt', 'main/iw_00.iwd', 'zone/english/a.ff']);
|
|
expect(ready.files[0].sha256).toBe(sha(Buffer.alloc(0)));
|
|
expect(ready.manifestSha256).toBe(manifestChecksum(ready.files));
|
|
expect(await readFile(join(ready.storageLocation, 'zone/english/a.ff'))).toEqual(bytes);
|
|
});
|
|
|
|
it('rejects incomplete publication and allows the remaining files to finish the same version', async () => {
|
|
const initial = begin('t5', [file('zone/first.ff'), file('zone/second.ff')]);
|
|
await upload(initial.id, 'zone/first.ff', bytes);
|
|
await expect(publishBrowserUpload(initial.id)).rejects.toThrow();
|
|
expect(getLibraryVersion(initial.id)?.state).not.toBe('ready');
|
|
const hidden = await serveGameFile(new Request('https://ec.test/files'), initial.id, 'zone/first.ff');
|
|
expect(hidden.status).toBe(404);
|
|
await upload(initial.id, 'zone/second.ff', bytes);
|
|
const ready = await publishBrowserUpload(initial.id);
|
|
expect(ready.state).toBe('ready');
|
|
expect(ready.id).toBe(initial.id);
|
|
});
|
|
|
|
it('does not acknowledge corrupt chunks and permits a verified retry', async () => {
|
|
const initial = begin();
|
|
await expect(upload(initial.id, file().path, bytes, 0, '0'.repeat(64))).rejects.toThrow();
|
|
expect((await browserUploadStatus(initial.id)).files[0]).toMatchObject({ receivedBytes: 0, complete: false });
|
|
await expect(uploadBrowserFile(initial.id, file().path, {
|
|
offset: 0, size: bytes.length, sha256: sha(bytes)
|
|
}, new Response(bytes.subarray(0, -1)).body)).rejects.toThrow();
|
|
expect((await browserUploadStatus(initial.id)).files[0].receivedBytes).toBe(0);
|
|
await upload(initial.id, file().path, bytes);
|
|
expect((await browserUploadStatus(initial.id)).files[0]).toMatchObject({ receivedBytes: bytes.length, complete: true });
|
|
expect((await publishBrowserUpload(initial.id)).state).toBe('ready');
|
|
});
|
|
|
|
it('surfaces asynchronous incomplete publication as retryable failure', async () => {
|
|
const initial = begin();
|
|
await requestBrowserPublication(initial.id);
|
|
await vi.waitFor(async () => expect((await browserUploadStatus(initial.id)).status).toBe('failed'));
|
|
expect((await browserUploadStatus(initial.id)).error).toBeTruthy();
|
|
await upload(initial.id, file().path, bytes);
|
|
expect(await browserUploadStatus(initial.id)).toMatchObject({ status: 'pending', error: null });
|
|
await requestBrowserPublication(initial.id);
|
|
expect((await publishBrowserUpload(initial.id)).state).toBe('ready');
|
|
});
|
|
|
|
it('resumes from the verified chunk boundary and safely retries a lost acknowledgement', async () => {
|
|
const sizeProbe = begin();
|
|
const chunkSize = (await browserUploadStatus(sizeProbe.id)).chunkSize;
|
|
expect(chunkSize).toBeGreaterThan(0);
|
|
const large = Buffer.alloc(chunkSize + 17, 0x41);
|
|
const initial = begin('t6', [file('main/archive.iwd', large)]);
|
|
const first = large.subarray(0, chunkSize);
|
|
await upload(initial.id, 'main/archive.iwd', first);
|
|
expect((await browserUploadStatus(initial.id)).files[0]).toMatchObject({ receivedBytes: chunkSize, complete: false });
|
|
state.handle!.close();
|
|
state.handle = new Database(join(directory, 'controller.sqlite')); initializeStandaloneSettings(state.handle);
|
|
state.handle.pragma('foreign_keys=ON');
|
|
await expect(upload(initial.id, 'main/archive.iwd', first)).rejects.toMatchObject({ receivedBytes: chunkSize });
|
|
expect((await browserUploadStatus(initial.id)).files[0].receivedBytes).toBe(chunkSize);
|
|
await upload(initial.id, 'main/archive.iwd', large.subarray(chunkSize), chunkSize);
|
|
const ready = await publishBrowserUpload(initial.id);
|
|
expect(ready.files[0].sha256).toBe(sha(large));
|
|
expect((await readFile(join(ready.storageLocation, 'main/archive.iwd'))).equals(large)).toBe(true);
|
|
});
|
|
|
|
it('rejects unknown file paths, traversal, and chunks outside the declared file', async () => {
|
|
const initial = begin();
|
|
await expect(upload(initial.id, 'zone/unselected.ff', bytes)).rejects.toThrow();
|
|
await expect(upload(initial.id, '../escape.ff', bytes)).rejects.toThrow();
|
|
await expect(upload(initial.id, file().path, bytes, bytes.length + 1)).rejects.toThrow();
|
|
await expect(upload(initial.id, file().path, Buffer.concat([bytes, bytes]))).rejects.toThrow();
|
|
expect((await browserUploadStatus(initial.id)).files[0].receivedBytes).toBe(0);
|
|
});
|
|
|
|
it('keeps published files immutable while accepting repeated uploads without labels', async () => {
|
|
const initial = begin(); await upload(initial.id, file().path, bytes);
|
|
const ready = await publishBrowserUpload(initial.id);
|
|
await expect(upload(initial.id, file().path, bytes)).rejects.toThrow();
|
|
expect(() => state.handle!.prepare('UPDATE game_file_versions SET description=? WHERE id=?').run('changed', ready.id)).toThrow(/immutable/);
|
|
expect((await publishBrowserUpload(initial.id)).manifestSha256).toBe(ready.manifestSha256);
|
|
const next = begin();
|
|
expect(next.id).not.toBe(ready.id);
|
|
expect(next.version).not.toBe(ready.version);
|
|
expect(next.libraryId).toBe(ready.libraryId);
|
|
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
|
|
});
|
|
|
|
it('ignores legacy names, descriptions, and labels when accepting new game files', () => {
|
|
const legacyFields = { name: 'same name', version: 'same version', description: 'a library description' };
|
|
const first = createLibraryUpload({ game: 't5', files: [file()], ...legacyFields });
|
|
const next = createLibraryUpload({ game: 't5', files: [file()], ...legacyFields });
|
|
const otherGame = createLibraryUpload({ game: 't6', files: [file()], ...legacyFields });
|
|
expect(first.description).toBe('');
|
|
expect(first.version).not.toBe(legacyFields.version);
|
|
expect(next.version).not.toBe(first.version);
|
|
expect(next.libraryId).toBe(first.libraryId);
|
|
expect(otherGame.libraryId).not.toBe(first.libraryId);
|
|
expect(listLibraries().map(library => library.name)).toEqual(['Black Ops I (T5)', 'Black Ops II (T6)']);
|
|
});
|
|
|
|
it('follows completion order and preserves the latest successful files during failed or unfinished uploads', async () => {
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(new Date('2026-01-01T00:00:00Z'));
|
|
const slow = begin();
|
|
vi.setSystemTime(new Date('2026-01-02T00:00:00Z'));
|
|
const fast = begin();
|
|
await upload(fast.id, file().path, bytes);
|
|
vi.setSystemTime(new Date('2026-01-03T00:00:00Z'));
|
|
await publishBrowserUpload(fast.id);
|
|
expect(newestReadyLibrary('t6')?.id).toBe(fast.id);
|
|
await upload(slow.id, file().path, bytes);
|
|
vi.setSystemTime(new Date('2026-01-04T00:00:00Z'));
|
|
await publishBrowserUpload(slow.id);
|
|
const failed = begin();
|
|
await requestBrowserPublication(failed.id);
|
|
await vi.waitFor(() => expect(getLibraryVersion(failed.id)?.state).toBe('failed'));
|
|
begin();
|
|
expect(newestReadyLibrary('t6')?.id).toBe(slow.id);
|
|
expect(latestPublishedLibraries().map(version => version.id)).toEqual([slow.id]);
|
|
expect(listLibraries().find(library => library.game === 't6')?.latestVersionId).toBe(slow.id);
|
|
});
|
|
|
|
it('recovers publication after the verified directory was renamed but the ready commit was interrupted', async () => {
|
|
const initial = begin(); await upload(initial.id, file().path, bytes);
|
|
const files = [{ path: file().path, size: bytes.length, sha256: sha(bytes) }];
|
|
const checksum = manifestChecksum(files);
|
|
state.handle!.transaction(() => {
|
|
state.handle!.prepare('UPDATE game_file_versions SET manifest=?,manifest_sha256=? WHERE id=?')
|
|
.run(JSON.stringify(files), checksum, initial.id);
|
|
state.handle!.prepare('INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)')
|
|
.run(initial.id, file().path, bytes.length, sha(bytes));
|
|
})();
|
|
await mkdir(dirname(initial.storageLocation), { recursive: true });
|
|
await rename(join(directory, 'libraries', '.imports', initial.id), initial.storageLocation);
|
|
state.handle!.close(); state.handle = new Database(join(directory, 'controller.sqlite')); initializeStandaloneSettings(state.handle);
|
|
state.handle.pragma('foreign_keys=ON');
|
|
expect(await browserUploadStatus(initial.id)).toMatchObject({ status: 'pending', files: [{ complete: true }] });
|
|
const ready = await publishBrowserUpload(initial.id);
|
|
expect(ready).toMatchObject({ id: initial.id, state: 'ready', manifestSha256: checksum, files });
|
|
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
|
|
});
|
|
|
|
it('rejects chunks racing with final publication without recreating staging files', async () => {
|
|
const initial = begin(); await upload(initial.id, file().path, bytes);
|
|
const publishing = publishBrowserUpload(initial.id);
|
|
await expect(upload(initial.id, file().path, bytes)).rejects.toMatchObject({ status: 409 });
|
|
const ready = await publishing;
|
|
expect(ready.state).toBe('ready');
|
|
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
|
|
await expect(stat(join(directory, 'libraries', '.imports', initial.id))).rejects.toMatchObject({ code: 'ENOENT' });
|
|
});
|
|
|
|
it('rejects browser operations against an existing host-sourced import', async () => {
|
|
const source = host();
|
|
const legacy = createLibraryImport({ hostId: source.id, game: 't6', name: 'Legacy import', version: '1' });
|
|
expect(legacy.sourceHostId).toBe(source.id);
|
|
await expect(Promise.resolve().then(() => browserUploadStatus(legacy.id))).rejects.toThrow();
|
|
await expect(upload(legacy.id, file().path, bytes)).rejects.toThrow();
|
|
await expect(publishBrowserUpload(legacy.id)).rejects.toThrow();
|
|
await expect(Promise.resolve().then(() => requestBrowserPublication(legacy.id))).rejects.toThrow();
|
|
expect(getLibraryVersion(legacy.id)?.state).toBe('importing');
|
|
});
|
|
|
|
it('distributes uploaded versions through the unchanged agent manifest, ranges, and verified cache contract', async () => {
|
|
const initial = begin('t5'); await upload(initial.id, file().path, bytes);
|
|
const ready = await publishBrowserUpload(initial.id);
|
|
const target = host();
|
|
const operation = queueLibraryDownload(target.id, ready.id);
|
|
expect(operation).toMatchObject({ kind: 'library.download', payload: {
|
|
libraryVersionId: ready.id, game: 't5', manifestSha256: ready.manifestSha256,
|
|
manifestUrl: `/api/agent/libraries/${ready.id}/manifest`, filesBaseUrl: `/api/agent/libraries/${ready.id}/files`
|
|
} });
|
|
const manifest = manifestRoute({ request: new Request(`https://ec.test/api/agent/libraries/${ready.id}/manifest`, {
|
|
headers: { Authorization: `Bearer ${target.credential}` }
|
|
}), params: { id: ready.id } });
|
|
expect(manifest.status).toBe(200);
|
|
expect(await manifest.json()).toEqual({ id: ready.id, game: 't5', files: ready.files, sizeBytes: bytes.length, manifestSha256: ready.manifestSha256 });
|
|
const range = await serveGameFile(new Request('https://ec.test/file', { headers: { Range: 'bytes=9-' } }), ready.id, file().path);
|
|
expect(range.status).toBe(206);
|
|
expect(Buffer.from(await range.arrayBuffer())).toEqual(bytes.subarray(9));
|
|
expect(() => requireReadyLibrary(target.id, ready.id)).toThrow(/finish downloading/);
|
|
updateHostLibraryCaches(target.id, [{ libraryVersionId: ready.id, status: 'ready', manifestSha256: '0'.repeat(64) }]);
|
|
expect(getHostLibraryCache(target.id, ready.id)?.status).toBe('failed');
|
|
updateHostLibraryCaches(target.id, [{ libraryVersionId: ready.id, status: 'ready', manifestSha256: ready.manifestSha256 }]);
|
|
expect(requireReadyLibrary(target.id, ready.id).id).toBe(ready.id);
|
|
const replacement = begin('t5'); await upload(replacement.id, file().path, bytes);
|
|
await publishBrowserUpload(replacement.id);
|
|
expect(requireReadyLibrary(target.id, ready.id).manifestSha256).toBe(ready.manifestSha256);
|
|
});
|
|
});
|
|
|
|
describe('game-file retention policies', () => {
|
|
beforeEach(() => state.handle!.exec('CREATE TABLE native_instances (id TEXT, host_id TEXT, library_id TEXT, retired_at TEXT)'));
|
|
it('expires inactive uploads while protecting resumed and explicitly held uploads', async () => {
|
|
const { fleetSettings,saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
const { cleanupControllerLibraries } = await import('../../src/lib/server/game-files/index.js');
|
|
saveFleetSettings({...fleetSettings(),uploadKeepDays:1});
|
|
const expired = begin(); const resumed = begin(); const held = begin();
|
|
for (const v of [expired,resumed,held]) await upload(v.id,file().path,bytes);
|
|
state.handle!.prepare("UPDATE game_file_versions SET last_activity_at='2020-01-01T00:00:00.000Z'").run();
|
|
state.handle!.prepare("INSERT INTO retention_controls(category,record_id,protected) VALUES('upload',?,1)").run(held.id);
|
|
await browserUploadStatus(resumed.id);
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(expired.id)?.filesRemoved).toBe(true);
|
|
await expect(upload(expired.id,file().path,bytes)).rejects.toThrow('expired');
|
|
expect((await browserUploadStatus(resumed.id)).files[0].complete).toBe(true);
|
|
expect((await browserUploadStatus(held.id)).files[0].complete).toBe(true);
|
|
});
|
|
it('keeps extra published versions and applies a supersession grace period', async () => {
|
|
const { fleetSettings,saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
const { cleanupControllerLibraries } = await import('../../src/lib/server/game-files/index.js');
|
|
saveFleetSettings({...fleetSettings(),gameFileKeepCount:2});
|
|
const first = begin(); await upload(first.id,file().path,bytes); await publishBrowserUpload(first.id);
|
|
const second = begin(); await upload(second.id,file().path,bytes); await publishBrowserUpload(second.id);
|
|
expect(getLibraryVersion(first.id)?.filesRemoved).toBe(false);
|
|
saveFleetSettings({...fleetSettings(),gameFileKeepCount:1,gameFileGraceDays:1});
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(first.id)?.filesRemoved).toBe(false);
|
|
saveFleetSettings({...fleetSettings(),gameFileGraceDays:0});
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(first.id)?.filesRemoved).toBe(true);
|
|
expect(getLibraryVersion(second.id)?.filesRemoved).toBe(false);
|
|
});
|
|
});
|
|
it('keeps an upload while its chunk is actively streaming, even if its activity timestamp is old', async () => {
|
|
const { fleetSettings,saveFleetSettings } = await import('../../src/lib/server/settings.js');
|
|
const { cleanupControllerLibraries } = await import('../../src/lib/server/game-files/index.js');
|
|
saveFleetSettings({...fleetSettings(),uploadKeepDays:1});
|
|
const version = begin();
|
|
let finish!: () => void;
|
|
const body = new ReadableStream<Uint8Array>({start(controller) {finish=()=>{controller.enqueue(bytes);controller.close();};}});
|
|
const writing = uploadBrowserFile(version.id,file().path,{offset:0,size:bytes.length,sha256:sha(bytes)},body);
|
|
try {
|
|
state.handle!.prepare("UPDATE game_file_versions SET last_activity_at='2020-01-01T00:00:00.000Z' WHERE id=?").run(version.id);
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(version.id)?.filesRemoving).toBe(false);
|
|
} finally {finish();await writing;}
|
|
expect((await browserUploadStatus(version.id)).files[0].complete).toBe(true);
|
|
});
|