329 lines
13 KiB
TypeScript
329 lines
13 KiB
TypeScript
import { createHash } from 'node:crypto';
|
|
import { mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
import {
|
|
getUploadFileProgress,
|
|
hashUploadedFiles,
|
|
MAX_UPLOAD_CHUNK_BYTES,
|
|
receiveUploadChunk,
|
|
UploadOffsetError,
|
|
validateUploadPlan,
|
|
withUploadLock,
|
|
withUploadFileLock
|
|
} from '../../src/lib/server/game-files/uploads.js';
|
|
import { manifestChecksum, verifyGameFile } from '../../src/lib/server/game-files/storage.js';
|
|
|
|
const sha = (value: string | Uint8Array) => createHash('sha256').update(value).digest('hex');
|
|
const stream = (value: string) => new Response(value).body!;
|
|
const file = { path: 'zone/english/map.ff', size: 10, lastModified: 1234 };
|
|
const chunk = (value: string, offset = 0) => ({
|
|
offset,
|
|
size: Buffer.byteLength(value),
|
|
sha256: sha(value)
|
|
});
|
|
let directory: string;
|
|
let root: string;
|
|
|
|
beforeEach(async () => {
|
|
directory = await mkdtemp(join(tmpdir(), 'ec-folder-upload-'));
|
|
root = join(directory, 'import');
|
|
});
|
|
afterEach(async () => {
|
|
await rm(directory, { recursive: true, force: true });
|
|
});
|
|
|
|
it('allows independent files together, serializes same-file offsets, and makes publication a barrier', async () => {
|
|
let release!: () => void;
|
|
const blocked = new Promise<void>((resolve) => {
|
|
release = resolve;
|
|
});
|
|
const order: string[] = [];
|
|
const first = withUploadFileLock(root, 'a', async () => {
|
|
order.push('a1');
|
|
await blocked;
|
|
});
|
|
const same = withUploadFileLock(root, 'a', async () => {
|
|
order.push('a2');
|
|
});
|
|
const other = withUploadFileLock(root, 'b', async () => {
|
|
order.push('b');
|
|
});
|
|
const publish = withUploadLock(root, async () => {
|
|
order.push('publish');
|
|
});
|
|
const later = withUploadFileLock(root, 'c', async () => {
|
|
order.push('c');
|
|
});
|
|
await other;
|
|
expect(order).toEqual(['a1', 'b']);
|
|
release();
|
|
await Promise.all([first, same, publish, later]);
|
|
expect(order).toEqual(['a1', 'b', 'a2', 'publish', 'c']);
|
|
});
|
|
|
|
describe('browser folder upload plans', () => {
|
|
it('preserves folder structure and source identity while sorting canonical paths', () => {
|
|
expect(
|
|
validateUploadPlan([
|
|
{ path: 'zone/map.ff', size: 10, lastModified: 100 },
|
|
{ path: 'main/data.iwd', size: 0 }
|
|
])
|
|
).toEqual([
|
|
{ path: 'main/data.iwd', size: 0, lastModified: 0 },
|
|
{ path: 'zone/map.ff', size: 10, lastModified: 100 }
|
|
]);
|
|
});
|
|
|
|
it.each(
|
|
[
|
|
[],
|
|
[{ path: '../zone.ff', size: 1 }],
|
|
[{ path: 'zone\\map.ff', size: 1 }],
|
|
[
|
|
{ path: 'Zone/a', size: 1 },
|
|
{ path: 'zone/A', size: 1 }
|
|
],
|
|
[
|
|
{ path: 'ZONE', size: 1 },
|
|
{ path: 'zone/a', size: 1 }
|
|
],
|
|
[{ path: 'zone/a', size: -1 }],
|
|
[{ path: 'zone/a', size: 1.5 }],
|
|
[
|
|
{ path: 'zone/a', size: Number.MAX_SAFE_INTEGER },
|
|
{ path: 'zone/b', size: 1 }
|
|
],
|
|
[{ path: 'zone/a', size: 1, lastModified: -1 }],
|
|
[{ path: 'zone/a', size: 1, lastModified: 1.5 }],
|
|
[{ path: 'zone/a', size: 1, lastModified: null }]
|
|
].map((plan) => ({ plan }))
|
|
)('rejects ambiguous or unsafe plan $plan', ({ plan }) => {
|
|
expect(() => validateUploadPlan(plan)).toThrow();
|
|
});
|
|
});
|
|
|
|
describe('bounded resumable file transfer', () => {
|
|
it('resumes arbitrary chunks by filesystem offset and produces a distribution-compatible manifest', async () => {
|
|
expect(await getUploadFileProgress(root, file)).toEqual({
|
|
receivedBytes: 0,
|
|
complete: false
|
|
});
|
|
expect(await receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).toEqual({
|
|
receivedBytes: 4,
|
|
complete: false
|
|
});
|
|
// A new request discovers its checkpoint from disk; no process-local receipt is required.
|
|
expect(await getUploadFileProgress(root, file)).toEqual({
|
|
receivedBytes: 4,
|
|
complete: false
|
|
});
|
|
expect(await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'))).toEqual({
|
|
receivedBytes: 10,
|
|
complete: true
|
|
});
|
|
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123456789');
|
|
const manifest = await withUploadLock(root, () => hashUploadedFiles(root, [file]));
|
|
expect(manifest).toEqual([{ path: file.path, size: 10, sha256: sha('0123456789') }]);
|
|
expect(manifestChecksum(manifest)).toBe(sha(JSON.stringify(manifest)));
|
|
await verifyGameFile(root, manifest[0]);
|
|
expect(await readdir(root)).toEqual(['zone']);
|
|
expect(await readdir(`${root}.chunks`)).toEqual([]);
|
|
});
|
|
|
|
it('reports committed progress after reply loss instead of appending duplicate bytes', async () => {
|
|
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
|
|
await expect(
|
|
receiveUploadChunk(root, file, chunk('0123'), stream('0123'))
|
|
).rejects.toMatchObject({
|
|
name: 'UploadOffsetError',
|
|
receivedBytes: 4,
|
|
status: 409
|
|
});
|
|
await expect(
|
|
receiveUploadChunk(root, file, chunk('89', 8), stream('89'))
|
|
).rejects.toBeInstanceOf(UploadOffsetError);
|
|
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
|
|
await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'));
|
|
});
|
|
|
|
it('can resume a verified append interrupted by process termination at an unaligned byte', async () => {
|
|
await mkdir(join(root, 'zone/english'), { recursive: true });
|
|
await writeFile(join(root, file.path), '0123456');
|
|
expect(await getUploadFileProgress(root, file)).toEqual({
|
|
receivedBytes: 7,
|
|
complete: false
|
|
});
|
|
await receiveUploadChunk(root, file, chunk('789', 7), stream('789'));
|
|
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123456789');
|
|
});
|
|
|
|
it('distinguishes an absent empty file and creates it using the empty digest', async () => {
|
|
const empty = { path: 'main/empty.dat', size: 0 };
|
|
expect(await getUploadFileProgress(root, empty)).toEqual({
|
|
receivedBytes: 0,
|
|
complete: false
|
|
});
|
|
await expect(hashUploadedFiles(root, [empty])).rejects.toThrow();
|
|
expect(await receiveUploadChunk(root, empty, chunk(''), null)).toEqual({
|
|
receivedBytes: 0,
|
|
complete: true
|
|
});
|
|
expect(await getUploadFileProgress(root, empty)).toEqual({
|
|
receivedBytes: 0,
|
|
complete: true
|
|
});
|
|
expect(await hashUploadedFiles(root, [empty])).toEqual([{ ...empty, sha256: sha('') }]);
|
|
});
|
|
|
|
it('does not advance progress for wrong checksums, truncated requests, oversized streams or disconnects', async () => {
|
|
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
|
|
await expect(
|
|
receiveUploadChunk(root, file, chunk('456789', 4), stream('xxxxxx'))
|
|
).rejects.toThrow(/checksum/);
|
|
await expect(
|
|
receiveUploadChunk(root, file, chunk('456789', 4), stream('45'))
|
|
).rejects.toThrow(/size mismatch/);
|
|
await expect(
|
|
receiveUploadChunk(root, file, chunk('456789', 4), stream('4567890'))
|
|
).rejects.toThrow(/exceeds/);
|
|
let reads = 0;
|
|
const disconnect = new ReadableStream<Uint8Array>({
|
|
pull(controller) {
|
|
if (reads++ === 0) controller.enqueue(new TextEncoder().encode('45'));
|
|
else controller.error(new Error('Browser connection ended.'));
|
|
}
|
|
});
|
|
await expect(
|
|
receiveUploadChunk(root, file, chunk('456789', 4), disconnect)
|
|
).rejects.toThrow(/connection ended/);
|
|
expect(await getUploadFileProgress(root, file)).toEqual({
|
|
receivedBytes: 4,
|
|
complete: false
|
|
});
|
|
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
|
|
expect(await readdir(`${root}.chunks`)).toEqual([]);
|
|
await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'));
|
|
});
|
|
|
|
it('bounds request size before reading any body and rejects invalid metadata', async () => {
|
|
const tooLarge = MAX_UPLOAD_CHUNK_BYTES + 1;
|
|
await expect(
|
|
receiveUploadChunk(
|
|
root,
|
|
{ path: 'large.iwd', size: tooLarge },
|
|
{
|
|
offset: 0,
|
|
size: tooLarge,
|
|
sha256: sha('')
|
|
},
|
|
null
|
|
)
|
|
).rejects.toThrow(/Invalid upload chunk/);
|
|
for (const metadata of [
|
|
{ ...chunk('a'), offset: -1 },
|
|
{ ...chunk('a'), offset: 0.5 },
|
|
{ ...chunk('a'), size: 1.5 },
|
|
{ ...chunk('a'), sha256: 'wrong' },
|
|
chunk('', 0),
|
|
chunk('01234567890', 0)
|
|
])
|
|
await expect(receiveUploadChunk(root, file, metadata, stream('a'))).rejects.toThrow(
|
|
/Invalid upload chunk/
|
|
);
|
|
await expect(receiveUploadChunk(root, file, chunk('a'), null)).rejects.toThrow(
|
|
/body is required/
|
|
);
|
|
});
|
|
|
|
it('streams a maximum-size chunk supplied in small pieces', async () => {
|
|
const piece = new Uint8Array(64 * 1024).fill(97);
|
|
let remaining = MAX_UPLOAD_CHUNK_BYTES;
|
|
const digest = createHash('sha256');
|
|
for (let i = 0; i < MAX_UPLOAD_CHUNK_BYTES / piece.length; i++) digest.update(piece);
|
|
const body = new ReadableStream<Uint8Array>({
|
|
pull(controller) {
|
|
if (remaining === 0) controller.close();
|
|
else {
|
|
remaining -= piece.length;
|
|
controller.enqueue(piece);
|
|
}
|
|
}
|
|
});
|
|
const large = { path: 'main/large.iwd', size: MAX_UPLOAD_CHUNK_BYTES };
|
|
const sha256 = digest.digest('hex');
|
|
await receiveUploadChunk(root, large, { offset: 0, size: large.size, sha256 }, body);
|
|
expect(await hashUploadedFiles(root, [large])).toEqual([{ ...large, sha256 }]);
|
|
});
|
|
|
|
it('rejects missing or incorrectly sized files before publication', async () => {
|
|
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
|
|
await expect(hashUploadedFiles(root, [file])).rejects.toThrow(/Size verification/);
|
|
await writeFile(join(root, file.path), '01234567890');
|
|
await expect(getUploadFileProgress(root, file)).rejects.toThrow(/exceeds/);
|
|
});
|
|
|
|
it('refuses traversal and symlinks in destination and temporary storage', async () => {
|
|
await mkdir(root);
|
|
const outside = join(directory, 'outside');
|
|
await mkdir(outside);
|
|
await expect(
|
|
receiveUploadChunk(root, { path: '../escape', size: 1 }, chunk('a'), stream('a'))
|
|
).rejects.toThrow(/relative/);
|
|
await symlink(outside, join(root, 'zone'));
|
|
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toThrow(
|
|
/non-regular/
|
|
);
|
|
await rm(join(root, 'zone'));
|
|
await symlink(outside, `${root}.chunks`);
|
|
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toThrow(
|
|
/non-regular/
|
|
);
|
|
expect(await readdir(outside)).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe('upload and publication coordination', () => {
|
|
it('serializes simultaneous writes with the same offset', async () => {
|
|
const responses = await Promise.allSettled([
|
|
receiveUploadChunk(root, file, chunk('0123'), stream('0123')),
|
|
receiveUploadChunk(root, file, chunk('0123'), stream('0123'))
|
|
]);
|
|
expect(responses.map((response) => response.status)).toEqual(['fulfilled', 'rejected']);
|
|
expect((responses[1] as PromiseRejectedResult).reason).toMatchObject({
|
|
name: 'UploadOffsetError',
|
|
receivedBytes: 4
|
|
});
|
|
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
|
|
});
|
|
|
|
it('rechecks writable state inside the lock after queued publication and releases failed locks', async () => {
|
|
let finishPublication!: () => void;
|
|
const gate = new Promise<void>((resolve) => {
|
|
finishPublication = resolve;
|
|
});
|
|
let published = false;
|
|
const publication = withUploadLock(root, async () => {
|
|
await gate;
|
|
published = true;
|
|
});
|
|
const transfer = receiveUploadChunk(root, file, chunk('0123'), stream('0123'), () => {
|
|
if (published) throw new Error('Library is immutable.');
|
|
});
|
|
finishPublication();
|
|
await publication;
|
|
await expect(transfer).rejects.toThrow(/immutable/);
|
|
expect(await getUploadFileProgress(root, file)).toEqual({
|
|
receivedBytes: 0,
|
|
complete: false
|
|
});
|
|
await expect(
|
|
withUploadLock(root, () => {
|
|
throw new Error('Failed work');
|
|
})
|
|
).rejects.toThrow(/Failed work/);
|
|
expect(await withUploadLock(root, () => 'next operation')).toBe('next operation');
|
|
});
|
|
});
|