197 lines
12 KiB
TypeScript
197 lines
12 KiB
TypeScript
import { createHash } from 'node:crypto';
|
|
import { mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
import {
|
|
getUploadFileProgress, hashUploadedFiles, MAX_UPLOAD_CHUNK_BYTES,
|
|
receiveUploadChunk, UploadOffsetError, validateUploadPlan, withUploadLock, withUploadFileLock
|
|
} from '../../src/lib/server/game-files/uploads.js';
|
|
import { manifestChecksum, verifyGameFile } from '../../src/lib/server/game-files/storage.js';
|
|
|
|
const sha = (value: string | Uint8Array) => createHash('sha256').update(value).digest('hex');
|
|
const stream = (value: string) => new Response(value).body!;
|
|
const file = { path: 'zone/english/map.ff', size: 10, lastModified: 1234 };
|
|
const chunk = (value: string, offset = 0) => ({ offset, size: Buffer.byteLength(value), sha256: sha(value) });
|
|
let directory: string;
|
|
let root: string;
|
|
|
|
beforeEach(async () => {
|
|
directory = await mkdtemp(join(tmpdir(), 'ec-folder-upload-'));
|
|
root = join(directory, 'import');
|
|
});
|
|
afterEach(async () => { await rm(directory, { recursive: true, force: true }); });
|
|
|
|
it('allows independent files together, serializes same-file offsets, and makes publication a barrier', async () => {
|
|
let release!: () => void;
|
|
const blocked = new Promise<void>(resolve => { release = resolve; });
|
|
const order: string[] = [];
|
|
const first = withUploadFileLock(root, 'a', async () => { order.push('a1'); await blocked; });
|
|
const same = withUploadFileLock(root, 'a', async () => { order.push('a2'); });
|
|
const other = withUploadFileLock(root, 'b', async () => { order.push('b'); });
|
|
const publish = withUploadLock(root, async () => { order.push('publish'); });
|
|
const later = withUploadFileLock(root, 'c', async () => { order.push('c'); });
|
|
await other;
|
|
expect(order).toEqual(['a1', 'b']);
|
|
release();
|
|
await Promise.all([first, same, publish, later]);
|
|
expect(order).toEqual(['a1', 'b', 'a2', 'publish', 'c']);
|
|
});
|
|
|
|
describe('browser folder upload plans', () => {
|
|
it('preserves folder structure and source identity while sorting canonical paths', () => {
|
|
expect(validateUploadPlan([{ path: 'zone/map.ff', size: 10, lastModified: 100 }, { path: 'main/data.iwd', size: 0 }])).toEqual([
|
|
{ path: 'main/data.iwd', size: 0, lastModified: 0 },
|
|
{ path: 'zone/map.ff', size: 10, lastModified: 100 }
|
|
]);
|
|
});
|
|
|
|
it.each([
|
|
[], [{ path: '../zone.ff', size: 1 }], [{ path: 'zone\\map.ff', size: 1 }],
|
|
[{ path: 'Zone/a', size: 1 }, { path: 'zone/A', size: 1 }],
|
|
[{ path: 'ZONE', size: 1 }, { path: 'zone/a', size: 1 }],
|
|
[{ path: 'zone/a', size: -1 }], [{ path: 'zone/a', size: 1.5 }],
|
|
[{ path: 'zone/a', size: Number.MAX_SAFE_INTEGER }, { path: 'zone/b', size: 1 }],
|
|
[{ path: 'zone/a', size: 1, lastModified: -1 }], [{ path: 'zone/a', size: 1, lastModified: 1.5 }],
|
|
[{ path: 'zone/a', size: 1, lastModified: null }]
|
|
].map(plan => ({ plan })))('rejects ambiguous or unsafe plan $plan', ({ plan }) => {
|
|
expect(() => validateUploadPlan(plan)).toThrow();
|
|
});
|
|
});
|
|
|
|
describe('bounded resumable file transfer', () => {
|
|
it('resumes arbitrary chunks by filesystem offset and produces a distribution-compatible manifest', async () => {
|
|
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 0, complete: false });
|
|
expect(await receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).toEqual({ receivedBytes: 4, complete: false });
|
|
// A new request discovers its checkpoint from disk; no process-local receipt is required.
|
|
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 4, complete: false });
|
|
expect(await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'))).toEqual({ receivedBytes: 10, complete: true });
|
|
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123456789');
|
|
const manifest = await withUploadLock(root, () => hashUploadedFiles(root, [file]));
|
|
expect(manifest).toEqual([{ path: file.path, size: 10, sha256: sha('0123456789') }]);
|
|
expect(manifestChecksum(manifest)).toBe(sha(JSON.stringify(manifest)));
|
|
await verifyGameFile(root, manifest[0]);
|
|
expect(await readdir(root)).toEqual(['zone']);
|
|
expect(await readdir(`${root}.chunks`)).toEqual([]);
|
|
});
|
|
|
|
it('reports committed progress after reply loss instead of appending duplicate bytes', async () => {
|
|
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
|
|
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toMatchObject({
|
|
name: 'UploadOffsetError', receivedBytes: 4, status: 409
|
|
});
|
|
await expect(receiveUploadChunk(root, file, chunk('89', 8), stream('89'))).rejects.toBeInstanceOf(UploadOffsetError);
|
|
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
|
|
await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'));
|
|
});
|
|
|
|
it('can resume a verified append interrupted by process termination at an unaligned byte', async () => {
|
|
await mkdir(join(root, 'zone/english'), { recursive: true });
|
|
await writeFile(join(root, file.path), '0123456');
|
|
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 7, complete: false });
|
|
await receiveUploadChunk(root, file, chunk('789', 7), stream('789'));
|
|
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123456789');
|
|
});
|
|
|
|
it('distinguishes an absent empty file and creates it using the empty digest', async () => {
|
|
const empty = { path: 'main/empty.dat', size: 0 };
|
|
expect(await getUploadFileProgress(root, empty)).toEqual({ receivedBytes: 0, complete: false });
|
|
await expect(hashUploadedFiles(root, [empty])).rejects.toThrow();
|
|
expect(await receiveUploadChunk(root, empty, chunk(''), null)).toEqual({ receivedBytes: 0, complete: true });
|
|
expect(await getUploadFileProgress(root, empty)).toEqual({ receivedBytes: 0, complete: true });
|
|
expect(await hashUploadedFiles(root, [empty])).toEqual([{ ...empty, sha256: sha('') }]);
|
|
});
|
|
|
|
it('does not advance progress for wrong checksums, truncated requests, oversized streams or disconnects', async () => {
|
|
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
|
|
await expect(receiveUploadChunk(root, file, chunk('456789', 4), stream('xxxxxx'))).rejects.toThrow(/checksum/);
|
|
await expect(receiveUploadChunk(root, file, chunk('456789', 4), stream('45'))).rejects.toThrow(/size mismatch/);
|
|
await expect(receiveUploadChunk(root, file, chunk('456789', 4), stream('4567890'))).rejects.toThrow(/exceeds/);
|
|
let reads = 0;
|
|
const disconnect = new ReadableStream<Uint8Array>({ pull(controller) {
|
|
if (reads++ === 0) controller.enqueue(new TextEncoder().encode('45'));
|
|
else controller.error(new Error('Browser connection ended.'));
|
|
} });
|
|
await expect(receiveUploadChunk(root, file, chunk('456789', 4), disconnect)).rejects.toThrow(/connection ended/);
|
|
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 4, complete: false });
|
|
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
|
|
expect(await readdir(`${root}.chunks`)).toEqual([]);
|
|
await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'));
|
|
});
|
|
|
|
it('bounds request size before reading any body and rejects invalid metadata', async () => {
|
|
const tooLarge = MAX_UPLOAD_CHUNK_BYTES + 1;
|
|
await expect(receiveUploadChunk(root, { path: 'large.iwd', size: tooLarge }, {
|
|
offset: 0, size: tooLarge, sha256: sha('')
|
|
}, null)).rejects.toThrow(/Invalid upload chunk/);
|
|
for (const metadata of [
|
|
{ ...chunk('a'), offset: -1 }, { ...chunk('a'), offset: 0.5 }, { ...chunk('a'), size: 1.5 },
|
|
{ ...chunk('a'), sha256: 'wrong' }, chunk('', 0), chunk('01234567890', 0)
|
|
]) await expect(receiveUploadChunk(root, file, metadata, stream('a'))).rejects.toThrow(/Invalid upload chunk/);
|
|
await expect(receiveUploadChunk(root, file, chunk('a'), null)).rejects.toThrow(/body is required/);
|
|
});
|
|
|
|
it('streams a maximum-size chunk supplied in small pieces', async () => {
|
|
const piece = new Uint8Array(64 * 1024).fill(97);
|
|
let remaining = MAX_UPLOAD_CHUNK_BYTES;
|
|
const digest = createHash('sha256');
|
|
for (let i = 0; i < MAX_UPLOAD_CHUNK_BYTES / piece.length; i++) digest.update(piece);
|
|
const body = new ReadableStream<Uint8Array>({ pull(controller) {
|
|
if (remaining === 0) controller.close();
|
|
else { remaining -= piece.length; controller.enqueue(piece); }
|
|
} });
|
|
const large = { path: 'main/large.iwd', size: MAX_UPLOAD_CHUNK_BYTES };
|
|
const sha256 = digest.digest('hex');
|
|
await receiveUploadChunk(root, large, { offset: 0, size: large.size, sha256 }, body);
|
|
expect(await hashUploadedFiles(root, [large])).toEqual([{ ...large, sha256 }]);
|
|
});
|
|
|
|
it('rejects missing or incorrectly sized files before publication', async () => {
|
|
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
|
|
await expect(hashUploadedFiles(root, [file])).rejects.toThrow(/Size verification/);
|
|
await writeFile(join(root, file.path), '01234567890');
|
|
await expect(getUploadFileProgress(root, file)).rejects.toThrow(/exceeds/);
|
|
});
|
|
|
|
it('refuses traversal and symlinks in destination and temporary storage', async () => {
|
|
await mkdir(root);
|
|
const outside = join(directory, 'outside');
|
|
await mkdir(outside);
|
|
await expect(receiveUploadChunk(root, { path: '../escape', size: 1 }, chunk('a'), stream('a'))).rejects.toThrow(/relative/);
|
|
await symlink(outside, join(root, 'zone'));
|
|
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toThrow(/non-regular/);
|
|
await rm(join(root, 'zone'));
|
|
await symlink(outside, `${root}.chunks`);
|
|
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toThrow(/non-regular/);
|
|
expect(await readdir(outside)).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe('upload and publication coordination', () => {
|
|
it('serializes simultaneous writes with the same offset', async () => {
|
|
const responses = await Promise.allSettled([
|
|
receiveUploadChunk(root, file, chunk('0123'), stream('0123')),
|
|
receiveUploadChunk(root, file, chunk('0123'), stream('0123'))
|
|
]);
|
|
expect(responses.map(response => response.status)).toEqual(['fulfilled', 'rejected']);
|
|
expect((responses[1] as PromiseRejectedResult).reason).toMatchObject({ name: 'UploadOffsetError', receivedBytes: 4 });
|
|
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
|
|
});
|
|
|
|
it('rechecks writable state inside the lock after queued publication and releases failed locks', async () => {
|
|
let finishPublication!: () => void;
|
|
const gate = new Promise<void>(resolve => { finishPublication = resolve; });
|
|
let published = false;
|
|
const publication = withUploadLock(root, async () => { await gate; published = true; });
|
|
const transfer = receiveUploadChunk(root, file, chunk('0123'), stream('0123'), () => {
|
|
if (published) throw new Error('Library is immutable.');
|
|
});
|
|
finishPublication();
|
|
await publication;
|
|
await expect(transfer).rejects.toThrow(/immutable/);
|
|
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 0, complete: false });
|
|
await expect(withUploadLock(root, () => { throw new Error('Failed work'); })).rejects.toThrow(/Failed work/);
|
|
expect(await withUploadLock(root, () => 'next operation')).toBe('next operation');
|
|
});
|
|
});
|