Files
electric-cherry/tests/unit/game-file-browser-upload.test.ts
T

510 lines
23 KiB
TypeScript

import { initializeStandaloneSettings } from '../helpers/settings.js';
import Database from 'better-sqlite3';
import { createHash } from 'node:crypto';
import { mkdir, mkdtemp, readFile, rename, rm, stat } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const state = vi.hoisted(() => ({ handle: null as Database.Database | null }));
vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle }));
import {
browserUploadStatus,
createLibraryImport,
createLibraryUpload,
getHostLibraryCache,
getLibraryVersion,
latestPublishedLibraries,
listLibraries,
newestReadyLibrary,
publishBrowserUpload,
queueLibraryDownload,
requestBrowserPublication,
requireReadyLibrary,
updateHostLibraryCaches,
uploadBrowserFile
} from '../../src/lib/server/game-files/index.js';
import {
createEnrollment,
createHost,
enrollHost,
listHosts
} from '../../src/lib/server/hosts/index.js';
import { manifestChecksum } from '../../src/lib/server/game-files/storage.js';
import { serveGameFile } from '../../src/lib/server/game-files/http.js';
import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js';
let directory: string;
const modified = 1_700_000_000_000;
const bytes = Buffer.from('verified Plutonium game content');
const sha = (value: Uint8Array) => createHash('sha256').update(value).digest('hex');
const file = (path = 'zone/base.ff', content = bytes) => ({
path,
size: content.length,
lastModified: modified
});
const upload = (id: string, path: string, content: Buffer, offset = 0, checksum = sha(content)) =>
uploadBrowserFile(
id,
path,
{ offset, size: content.length, sha256: checksum },
new Response(new Uint8Array(content)).body
);
const begin = (game: 't5' | 't6' = 't6', files = [file()]) => createLibraryUpload({ game, files });
beforeEach(async () => {
directory = await mkdtemp(join(tmpdir(), 'ec-browser-library-'));
state.handle = new Database(join(directory, 'controller.sqlite'));
initializeStandaloneSettings(state.handle);
state.handle.pragma('foreign_keys=ON');
vi.stubEnv('GAME_FILES_ROOT', join(directory, 'libraries'));
});
afterEach(async () => {
state.handle?.close();
state.handle = null;
await rm(directory, { recursive: true, force: true });
vi.unstubAllEnvs();
vi.useRealTimers();
});
function host() {
const created = createHost({ name: 'Fresh download target' });
const { credential } = enrollHost(createEnrollment(created.id).token, {
protocolVersion: 1,
dockerRunning: true,
directoriesWritable: true,
storageAvailable: 100 * 1024 ** 3
});
return { id: created.id, credential };
}
describe('browser folder uploads', () => {
it.each(['t5', 't6'] as const)(
'publishes %s from a browser without an enrolled host or DLC declarations',
async (game) => {
expect(listHosts()).toEqual([]);
const initial = begin(game);
expect(initial).toMatchObject({
state: 'importing',
operationId: null,
sourceHostId: null,
sourceVolume: null
});
expect(initial).not.toHaveProperty('dlc');
expect(await browserUploadStatus(initial.id)).toMatchObject({
status: 'pending',
files: [{ ...file(), receivedBytes: 0, complete: false }]
});
await upload(initial.id, file().path, bytes);
const accepted = await requestBrowserPublication(initial.id);
expect(['verifying', 'published']).toContain(accepted.status);
const ready = await publishBrowserUpload(initial.id);
expect(ready).toMatchObject({ state: 'ready', sizeBytes: bytes.length, fileCount: 1 });
expect(ready.files).toEqual([
{ path: file().path, size: bytes.length, sha256: sha(bytes) }
]);
expect(ready.manifestSha256).toBe(manifestChecksum(ready.files));
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
expect(await browserUploadStatus(initial.id)).toMatchObject({ status: 'published' });
expect(
listLibraries().find((library) => library.game === game)?.versions[0]
).not.toHaveProperty('dlc');
expect(listHosts()).toEqual([]);
}
);
it('preserves nested relative contents and derives a canonical manifest in path order', async () => {
const selected = [
file('zone/english/a.ff'),
file('main/iw_00.iwd'),
file('empty.txt', Buffer.alloc(0))
];
const initial = begin('t6', selected);
for (const item of selected)
await upload(initial.id, item.path, item.size ? bytes : Buffer.alloc(0));
const ready = await publishBrowserUpload(initial.id);
expect(ready.files.map((item) => item.path)).toEqual([
'empty.txt',
'main/iw_00.iwd',
'zone/english/a.ff'
]);
expect(ready.files[0].sha256).toBe(sha(Buffer.alloc(0)));
expect(ready.manifestSha256).toBe(manifestChecksum(ready.files));
expect(await readFile(join(ready.storageLocation, 'zone/english/a.ff'))).toEqual(bytes);
});
it('rejects incomplete publication and allows the remaining files to finish the same version', async () => {
const initial = begin('t5', [file('zone/first.ff'), file('zone/second.ff')]);
await upload(initial.id, 'zone/first.ff', bytes);
await expect(publishBrowserUpload(initial.id)).rejects.toThrow();
expect(getLibraryVersion(initial.id)?.state).not.toBe('ready');
const hidden = await serveGameFile(
new Request('https://ec.test/files'),
initial.id,
'zone/first.ff'
);
expect(hidden.status).toBe(404);
await upload(initial.id, 'zone/second.ff', bytes);
const ready = await publishBrowserUpload(initial.id);
expect(ready.state).toBe('ready');
expect(ready.id).toBe(initial.id);
});
it('does not acknowledge corrupt chunks and permits a verified retry', async () => {
const initial = begin();
await expect(upload(initial.id, file().path, bytes, 0, '0'.repeat(64))).rejects.toThrow();
expect((await browserUploadStatus(initial.id)).files[0]).toMatchObject({
receivedBytes: 0,
complete: false
});
await expect(
uploadBrowserFile(
initial.id,
file().path,
{
offset: 0,
size: bytes.length,
sha256: sha(bytes)
},
new Response(bytes.subarray(0, -1)).body
)
).rejects.toThrow();
expect((await browserUploadStatus(initial.id)).files[0].receivedBytes).toBe(0);
await upload(initial.id, file().path, bytes);
expect((await browserUploadStatus(initial.id)).files[0]).toMatchObject({
receivedBytes: bytes.length,
complete: true
});
expect((await publishBrowserUpload(initial.id)).state).toBe('ready');
});
it('surfaces asynchronous incomplete publication as retryable failure', async () => {
const initial = begin();
await requestBrowserPublication(initial.id);
await vi.waitFor(async () =>
expect((await browserUploadStatus(initial.id)).status).toBe('failed')
);
expect((await browserUploadStatus(initial.id)).error).toBeTruthy();
await upload(initial.id, file().path, bytes);
expect(await browserUploadStatus(initial.id)).toMatchObject({
status: 'pending',
error: null
});
await requestBrowserPublication(initial.id);
expect((await publishBrowserUpload(initial.id)).state).toBe('ready');
});
it('resumes from the verified chunk boundary and safely retries a lost acknowledgement', async () => {
const sizeProbe = begin();
const chunkSize = (await browserUploadStatus(sizeProbe.id)).chunkSize;
expect(chunkSize).toBeGreaterThan(0);
const large = Buffer.alloc(chunkSize + 17, 0x41);
const initial = begin('t6', [file('main/archive.iwd', large)]);
const first = large.subarray(0, chunkSize);
await upload(initial.id, 'main/archive.iwd', first);
expect((await browserUploadStatus(initial.id)).files[0]).toMatchObject({
receivedBytes: chunkSize,
complete: false
});
state.handle!.close();
state.handle = new Database(join(directory, 'controller.sqlite'));
initializeStandaloneSettings(state.handle);
state.handle.pragma('foreign_keys=ON');
await expect(upload(initial.id, 'main/archive.iwd', first)).rejects.toMatchObject({
receivedBytes: chunkSize
});
expect((await browserUploadStatus(initial.id)).files[0].receivedBytes).toBe(chunkSize);
await upload(initial.id, 'main/archive.iwd', large.subarray(chunkSize), chunkSize);
const ready = await publishBrowserUpload(initial.id);
expect(ready.files[0].sha256).toBe(sha(large));
expect(
(await readFile(join(ready.storageLocation, 'main/archive.iwd'))).equals(large)
).toBe(true);
});
it('rejects unknown file paths, traversal, and chunks outside the declared file', async () => {
const initial = begin();
await expect(upload(initial.id, 'zone/unselected.ff', bytes)).rejects.toThrow();
await expect(upload(initial.id, '../escape.ff', bytes)).rejects.toThrow();
await expect(upload(initial.id, file().path, bytes, bytes.length + 1)).rejects.toThrow();
await expect(
upload(initial.id, file().path, Buffer.concat([bytes, bytes]))
).rejects.toThrow();
expect((await browserUploadStatus(initial.id)).files[0].receivedBytes).toBe(0);
});
it('keeps published files immutable while accepting repeated uploads without labels', async () => {
const initial = begin();
await upload(initial.id, file().path, bytes);
const ready = await publishBrowserUpload(initial.id);
await expect(upload(initial.id, file().path, bytes)).rejects.toThrow();
expect(() =>
state
.handle!.prepare('UPDATE game_file_versions SET description=? WHERE id=?')
.run('changed', ready.id)
).toThrow(/immutable/);
expect((await publishBrowserUpload(initial.id)).manifestSha256).toBe(ready.manifestSha256);
const next = begin();
expect(next.id).not.toBe(ready.id);
expect(next.version).not.toBe(ready.version);
expect(next.libraryId).toBe(ready.libraryId);
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
});
it('ignores legacy names, descriptions, and labels when accepting new game files', () => {
const legacyFields = {
name: 'same name',
version: 'same version',
description: 'a library description'
};
const first = createLibraryUpload({ game: 't5', files: [file()], ...legacyFields });
const next = createLibraryUpload({ game: 't5', files: [file()], ...legacyFields });
const otherGame = createLibraryUpload({ game: 't6', files: [file()], ...legacyFields });
expect(first.description).toBe('');
expect(first.version).not.toBe(legacyFields.version);
expect(next.version).not.toBe(first.version);
expect(next.libraryId).toBe(first.libraryId);
expect(otherGame.libraryId).not.toBe(first.libraryId);
expect(listLibraries().map((library) => library.name)).toEqual([
'Black Ops I',
'Black Ops II'
]);
});
it('follows completion order and preserves the latest successful files during failed or unfinished uploads', async () => {
vi.useFakeTimers({ toFake: ['Date'] });
vi.setSystemTime(new Date('2026-01-01T00:00:00Z'));
const slow = begin();
vi.setSystemTime(new Date('2026-01-02T00:00:00Z'));
const fast = begin();
await upload(fast.id, file().path, bytes);
vi.setSystemTime(new Date('2026-01-03T00:00:00Z'));
await publishBrowserUpload(fast.id);
expect(newestReadyLibrary('t6')?.id).toBe(fast.id);
await upload(slow.id, file().path, bytes);
vi.setSystemTime(new Date('2026-01-04T00:00:00Z'));
await publishBrowserUpload(slow.id);
const failed = begin();
await requestBrowserPublication(failed.id);
await vi.waitFor(() => expect(getLibraryVersion(failed.id)?.state).toBe('failed'));
begin();
expect(newestReadyLibrary('t6')?.id).toBe(slow.id);
expect(latestPublishedLibraries().map((version) => version.id)).toEqual([slow.id]);
expect(listLibraries().find((library) => library.game === 't6')?.latestVersionId).toBe(
slow.id
);
});
it('recovers publication after the verified directory was renamed but the ready commit was interrupted', async () => {
const initial = begin();
await upload(initial.id, file().path, bytes);
const files = [{ path: file().path, size: bytes.length, sha256: sha(bytes) }];
const checksum = manifestChecksum(files);
state.handle!.transaction(() => {
state
.handle!.prepare(
'UPDATE game_file_versions SET manifest=?,manifest_sha256=? WHERE id=?'
)
.run(JSON.stringify(files), checksum, initial.id);
state
.handle!.prepare(
'INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)'
)
.run(initial.id, file().path, bytes.length, sha(bytes));
})();
await mkdir(dirname(initial.storageLocation), { recursive: true });
await rename(join(directory, 'libraries', '.imports', initial.id), initial.storageLocation);
state.handle!.close();
state.handle = new Database(join(directory, 'controller.sqlite'));
initializeStandaloneSettings(state.handle);
state.handle.pragma('foreign_keys=ON');
expect(await browserUploadStatus(initial.id)).toMatchObject({
status: 'pending',
files: [{ complete: true }]
});
const ready = await publishBrowserUpload(initial.id);
expect(ready).toMatchObject({
id: initial.id,
state: 'ready',
manifestSha256: checksum,
files
});
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
});
it('rejects chunks racing with final publication without recreating staging files', async () => {
const initial = begin();
await upload(initial.id, file().path, bytes);
const publishing = publishBrowserUpload(initial.id);
await expect(upload(initial.id, file().path, bytes)).rejects.toMatchObject({ status: 409 });
const ready = await publishing;
expect(ready.state).toBe('ready');
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
await expect(
stat(join(directory, 'libraries', '.imports', initial.id))
).rejects.toMatchObject({ code: 'ENOENT' });
});
it('rejects browser operations against an existing host-sourced import', async () => {
const source = host();
const legacy = createLibraryImport({
hostId: source.id,
game: 't6',
name: 'Legacy import',
version: '1'
});
expect(legacy.sourceHostId).toBe(source.id);
await expect(
Promise.resolve().then(() => browserUploadStatus(legacy.id))
).rejects.toThrow();
await expect(upload(legacy.id, file().path, bytes)).rejects.toThrow();
await expect(publishBrowserUpload(legacy.id)).rejects.toThrow();
await expect(
Promise.resolve().then(() => requestBrowserPublication(legacy.id))
).rejects.toThrow();
expect(getLibraryVersion(legacy.id)?.state).toBe('importing');
});
it('distributes uploaded versions through the unchanged agent manifest, ranges, and verified cache contract', async () => {
const initial = begin('t5');
await upload(initial.id, file().path, bytes);
const ready = await publishBrowserUpload(initial.id);
const target = host();
const operation = queueLibraryDownload(target.id, ready.id);
expect(operation).toMatchObject({
kind: 'library.download',
payload: {
libraryVersionId: ready.id,
game: 't5',
manifestSha256: ready.manifestSha256,
manifestUrl: `/api/agent/libraries/${ready.id}/manifest`,
filesBaseUrl: `/api/agent/libraries/${ready.id}/files`
}
});
const manifest = manifestRoute({
request: new Request(`https://ec.test/api/agent/libraries/${ready.id}/manifest`, {
headers: { Authorization: `Bearer ${target.credential}` }
}),
params: { id: ready.id }
});
expect(manifest.status).toBe(200);
expect(await manifest.json()).toEqual({
id: ready.id,
game: 't5',
files: ready.files,
sizeBytes: bytes.length,
manifestSha256: ready.manifestSha256
});
const range = await serveGameFile(
new Request('https://ec.test/file', { headers: { Range: 'bytes=9-' } }),
ready.id,
file().path
);
expect(range.status).toBe(206);
expect(Buffer.from(await range.arrayBuffer())).toEqual(bytes.subarray(9));
expect(() => requireReadyLibrary(target.id, ready.id)).toThrow(/finish downloading/);
updateHostLibraryCaches(target.id, [
{ libraryVersionId: ready.id, status: 'ready', manifestSha256: '0'.repeat(64) }
]);
expect(getHostLibraryCache(target.id, ready.id)?.status).toBe('failed');
updateHostLibraryCaches(target.id, [
{ libraryVersionId: ready.id, status: 'ready', manifestSha256: ready.manifestSha256 }
]);
expect(requireReadyLibrary(target.id, ready.id).id).toBe(ready.id);
const replacement = begin('t5');
await upload(replacement.id, file().path, bytes);
await publishBrowserUpload(replacement.id);
expect(requireReadyLibrary(target.id, ready.id).manifestSha256).toBe(ready.manifestSha256);
});
});
describe('game-file retention policies', () => {
beforeEach(() =>
state.handle!.exec(
'CREATE TABLE native_instances (id TEXT, host_id TEXT, library_id TEXT, retired_at TEXT)'
)
);
it('expires inactive uploads while protecting resumed and explicitly held uploads', async () => {
const { fleetSettings, saveFleetSettings } =
await import('../../src/lib/server/settings.js');
const { cleanupControllerLibraries } =
await import('../../src/lib/server/game-files/index.js');
saveFleetSettings({ ...fleetSettings(), uploadKeepDays: 1 });
const expired = begin();
const resumed = begin();
const held = begin();
for (const v of [expired, resumed, held]) await upload(v.id, file().path, bytes);
state
.handle!.prepare(
"UPDATE game_file_versions SET last_activity_at='2020-01-01T00:00:00.000Z'"
)
.run();
state
.handle!.prepare(
"INSERT INTO retention_controls(category,record_id,protected) VALUES('upload',?,1)"
)
.run(held.id);
await browserUploadStatus(resumed.id);
await cleanupControllerLibraries();
expect(getLibraryVersion(expired.id)?.filesRemoved).toBe(true);
await expect(upload(expired.id, file().path, bytes)).rejects.toThrow('expired');
expect((await browserUploadStatus(resumed.id)).files[0].complete).toBe(true);
expect((await browserUploadStatus(held.id)).files[0].complete).toBe(true);
});
it('keeps extra published versions and applies a supersession grace period', async () => {
const { fleetSettings, saveFleetSettings } =
await import('../../src/lib/server/settings.js');
const { cleanupControllerLibraries } =
await import('../../src/lib/server/game-files/index.js');
saveFleetSettings({ ...fleetSettings(), gameFileKeepCount: 2 });
const first = begin();
await upload(first.id, file().path, bytes);
await publishBrowserUpload(first.id);
const second = begin();
await upload(second.id, file().path, bytes);
await publishBrowserUpload(second.id);
expect(getLibraryVersion(first.id)?.filesRemoved).toBe(false);
saveFleetSettings({ ...fleetSettings(), gameFileKeepCount: 1, gameFileGraceDays: 1 });
await cleanupControllerLibraries();
expect(getLibraryVersion(first.id)?.filesRemoved).toBe(false);
saveFleetSettings({ ...fleetSettings(), gameFileGraceDays: 0 });
await cleanupControllerLibraries();
expect(getLibraryVersion(first.id)?.filesRemoved).toBe(true);
expect(getLibraryVersion(second.id)?.filesRemoved).toBe(false);
});
});
it('keeps an upload while its chunk is actively streaming, even if its activity timestamp is old', async () => {
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
const { cleanupControllerLibraries } = await import('../../src/lib/server/game-files/index.js');
saveFleetSettings({ ...fleetSettings(), uploadKeepDays: 1 });
const version = begin();
let finish!: () => void;
const body = new ReadableStream<Uint8Array>({
start(controller) {
finish = () => {
controller.enqueue(bytes);
controller.close();
};
}
});
const writing = uploadBrowserFile(
version.id,
file().path,
{ offset: 0, size: bytes.length, sha256: sha(bytes) },
body
);
try {
state
.handle!.prepare(
"UPDATE game_file_versions SET last_activity_at='2020-01-01T00:00:00.000Z' WHERE id=?"
)
.run(version.id);
await cleanupControllerLibraries();
expect(getLibraryVersion(version.id)?.filesRemoving).toBe(false);
} finally {
finish();
await writing;
}
expect((await browserUploadStatus(version.id)).files[0].complete).toBe(true);
});