Support resumable game folder uploads and remove DLC declarations
This commit is contained in:
@@ -0,0 +1,180 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
getUploadFileProgress, hashUploadedFiles, MAX_UPLOAD_CHUNK_BYTES,
|
||||
receiveUploadChunk, UploadOffsetError, validateUploadPlan, withUploadLock
|
||||
} from '../../src/lib/server/game-files/uploads.js';
|
||||
import { manifestChecksum, verifyGameFile } from '../../src/lib/server/game-files/storage.js';
|
||||
|
||||
const sha = (value: string | Uint8Array) => createHash('sha256').update(value).digest('hex');
|
||||
const stream = (value: string) => new Response(value).body!;
|
||||
const file = { path: 'zone/english/map.ff', size: 10, lastModified: 1234 };
|
||||
const chunk = (value: string, offset = 0) => ({ offset, size: Buffer.byteLength(value), sha256: sha(value) });
|
||||
let directory: string;
|
||||
let root: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
directory = await mkdtemp(join(tmpdir(), 'ec-folder-upload-'));
|
||||
root = join(directory, 'import');
|
||||
});
|
||||
afterEach(async () => { await rm(directory, { recursive: true, force: true }); });
|
||||
|
||||
describe('browser folder upload plans', () => {
|
||||
it('preserves folder structure and source identity while sorting canonical paths', () => {
|
||||
expect(validateUploadPlan([{ path: 'zone/map.ff', size: 10, lastModified: 100 }, { path: 'main/data.iwd', size: 0 }])).toEqual([
|
||||
{ path: 'main/data.iwd', size: 0, lastModified: 0 },
|
||||
{ path: 'zone/map.ff', size: 10, lastModified: 100 }
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[], [{ path: '../zone.ff', size: 1 }], [{ path: 'zone\\map.ff', size: 1 }],
|
||||
[{ path: 'Zone/a', size: 1 }, { path: 'zone/A', size: 1 }],
|
||||
[{ path: 'ZONE', size: 1 }, { path: 'zone/a', size: 1 }],
|
||||
[{ path: 'zone/a', size: -1 }], [{ path: 'zone/a', size: 1.5 }],
|
||||
[{ path: 'zone/a', size: Number.MAX_SAFE_INTEGER }, { path: 'zone/b', size: 1 }],
|
||||
[{ path: 'zone/a', size: 1, lastModified: -1 }], [{ path: 'zone/a', size: 1, lastModified: 1.5 }],
|
||||
[{ path: 'zone/a', size: 1, lastModified: null }]
|
||||
].map(plan => ({ plan })))('rejects ambiguous or unsafe plan $plan', ({ plan }) => {
|
||||
expect(() => validateUploadPlan(plan)).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('bounded resumable file transfer', () => {
|
||||
it('resumes arbitrary chunks by filesystem offset and produces a distribution-compatible manifest', async () => {
|
||||
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 0, complete: false });
|
||||
expect(await receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).toEqual({ receivedBytes: 4, complete: false });
|
||||
// A new request discovers its checkpoint from disk; no process-local receipt is required.
|
||||
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 4, complete: false });
|
||||
expect(await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'))).toEqual({ receivedBytes: 10, complete: true });
|
||||
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123456789');
|
||||
const manifest = await withUploadLock(root, () => hashUploadedFiles(root, [file]));
|
||||
expect(manifest).toEqual([{ path: file.path, size: 10, sha256: sha('0123456789') }]);
|
||||
expect(manifestChecksum(manifest)).toBe(sha(JSON.stringify(manifest)));
|
||||
await verifyGameFile(root, manifest[0]);
|
||||
expect(await readdir(root)).toEqual(['zone']);
|
||||
expect(await readdir(`${root}.chunks`)).toEqual([]);
|
||||
});
|
||||
|
||||
it('reports committed progress after reply loss instead of appending duplicate bytes', async () => {
|
||||
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
|
||||
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toMatchObject({
|
||||
name: 'UploadOffsetError', receivedBytes: 4, status: 409
|
||||
});
|
||||
await expect(receiveUploadChunk(root, file, chunk('89', 8), stream('89'))).rejects.toBeInstanceOf(UploadOffsetError);
|
||||
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
|
||||
await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'));
|
||||
});
|
||||
|
||||
it('can resume a verified append interrupted by process termination at an unaligned byte', async () => {
|
||||
await mkdir(join(root, 'zone/english'), { recursive: true });
|
||||
await writeFile(join(root, file.path), '0123456');
|
||||
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 7, complete: false });
|
||||
await receiveUploadChunk(root, file, chunk('789', 7), stream('789'));
|
||||
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123456789');
|
||||
});
|
||||
|
||||
it('distinguishes an absent empty file and creates it using the empty digest', async () => {
|
||||
const empty = { path: 'main/empty.dat', size: 0 };
|
||||
expect(await getUploadFileProgress(root, empty)).toEqual({ receivedBytes: 0, complete: false });
|
||||
await expect(hashUploadedFiles(root, [empty])).rejects.toThrow();
|
||||
expect(await receiveUploadChunk(root, empty, chunk(''), null)).toEqual({ receivedBytes: 0, complete: true });
|
||||
expect(await getUploadFileProgress(root, empty)).toEqual({ receivedBytes: 0, complete: true });
|
||||
expect(await hashUploadedFiles(root, [empty])).toEqual([{ ...empty, sha256: sha('') }]);
|
||||
});
|
||||
|
||||
it('does not advance progress for wrong checksums, truncated requests, oversized streams or disconnects', async () => {
|
||||
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
|
||||
await expect(receiveUploadChunk(root, file, chunk('456789', 4), stream('xxxxxx'))).rejects.toThrow(/checksum/);
|
||||
await expect(receiveUploadChunk(root, file, chunk('456789', 4), stream('45'))).rejects.toThrow(/size mismatch/);
|
||||
await expect(receiveUploadChunk(root, file, chunk('456789', 4), stream('4567890'))).rejects.toThrow(/exceeds/);
|
||||
let reads = 0;
|
||||
const disconnect = new ReadableStream<Uint8Array>({ pull(controller) {
|
||||
if (reads++ === 0) controller.enqueue(new TextEncoder().encode('45'));
|
||||
else controller.error(new Error('Browser connection ended.'));
|
||||
} });
|
||||
await expect(receiveUploadChunk(root, file, chunk('456789', 4), disconnect)).rejects.toThrow(/connection ended/);
|
||||
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 4, complete: false });
|
||||
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
|
||||
expect(await readdir(`${root}.chunks`)).toEqual([]);
|
||||
await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'));
|
||||
});
|
||||
|
||||
it('bounds request size before reading any body and rejects invalid metadata', async () => {
|
||||
const tooLarge = MAX_UPLOAD_CHUNK_BYTES + 1;
|
||||
await expect(receiveUploadChunk(root, { path: 'large.iwd', size: tooLarge }, {
|
||||
offset: 0, size: tooLarge, sha256: sha('')
|
||||
}, null)).rejects.toThrow(/Invalid upload chunk/);
|
||||
for (const metadata of [
|
||||
{ ...chunk('a'), offset: -1 }, { ...chunk('a'), offset: 0.5 }, { ...chunk('a'), size: 1.5 },
|
||||
{ ...chunk('a'), sha256: 'wrong' }, chunk('', 0), chunk('01234567890', 0)
|
||||
]) await expect(receiveUploadChunk(root, file, metadata, stream('a'))).rejects.toThrow(/Invalid upload chunk/);
|
||||
await expect(receiveUploadChunk(root, file, chunk('a'), null)).rejects.toThrow(/body is required/);
|
||||
});
|
||||
|
||||
it('streams a maximum-size chunk supplied in small pieces', async () => {
|
||||
const piece = new Uint8Array(64 * 1024).fill(97);
|
||||
let remaining = MAX_UPLOAD_CHUNK_BYTES;
|
||||
const digest = createHash('sha256');
|
||||
for (let i = 0; i < MAX_UPLOAD_CHUNK_BYTES / piece.length; i++) digest.update(piece);
|
||||
const body = new ReadableStream<Uint8Array>({ pull(controller) {
|
||||
if (remaining === 0) controller.close();
|
||||
else { remaining -= piece.length; controller.enqueue(piece); }
|
||||
} });
|
||||
const large = { path: 'main/large.iwd', size: MAX_UPLOAD_CHUNK_BYTES };
|
||||
const sha256 = digest.digest('hex');
|
||||
await receiveUploadChunk(root, large, { offset: 0, size: large.size, sha256 }, body);
|
||||
expect(await hashUploadedFiles(root, [large])).toEqual([{ ...large, sha256 }]);
|
||||
});
|
||||
|
||||
it('rejects missing or incorrectly sized files before publication', async () => {
|
||||
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
|
||||
await expect(hashUploadedFiles(root, [file])).rejects.toThrow(/Size verification/);
|
||||
await writeFile(join(root, file.path), '01234567890');
|
||||
await expect(getUploadFileProgress(root, file)).rejects.toThrow(/exceeds/);
|
||||
});
|
||||
|
||||
it('refuses traversal and symlinks in destination and temporary storage', async () => {
|
||||
await mkdir(root);
|
||||
const outside = join(directory, 'outside');
|
||||
await mkdir(outside);
|
||||
await expect(receiveUploadChunk(root, { path: '../escape', size: 1 }, chunk('a'), stream('a'))).rejects.toThrow(/relative/);
|
||||
await symlink(outside, join(root, 'zone'));
|
||||
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toThrow(/non-regular/);
|
||||
await rm(join(root, 'zone'));
|
||||
await symlink(outside, `${root}.chunks`);
|
||||
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toThrow(/non-regular/);
|
||||
expect(await readdir(outside)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('upload and publication coordination', () => {
|
||||
it('serializes simultaneous writes with the same offset', async () => {
|
||||
const responses = await Promise.allSettled([
|
||||
receiveUploadChunk(root, file, chunk('0123'), stream('0123')),
|
||||
receiveUploadChunk(root, file, chunk('0123'), stream('0123'))
|
||||
]);
|
||||
expect(responses.map(response => response.status)).toEqual(['fulfilled', 'rejected']);
|
||||
expect((responses[1] as PromiseRejectedResult).reason).toMatchObject({ name: 'UploadOffsetError', receivedBytes: 4 });
|
||||
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
|
||||
});
|
||||
|
||||
it('rechecks writable state inside the lock after queued publication and releases failed locks', async () => {
|
||||
let finishPublication!: () => void;
|
||||
const gate = new Promise<void>(resolve => { finishPublication = resolve; });
|
||||
let published = false;
|
||||
const publication = withUploadLock(root, async () => { await gate; published = true; });
|
||||
const transfer = receiveUploadChunk(root, file, chunk('0123'), stream('0123'), () => {
|
||||
if (published) throw new Error('Library is immutable.');
|
||||
});
|
||||
finishPublication();
|
||||
await publication;
|
||||
await expect(transfer).rejects.toThrow(/immutable/);
|
||||
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 0, complete: false });
|
||||
await expect(withUploadLock(root, () => { throw new Error('Failed work'); })).rejects.toThrow(/Failed work/);
|
||||
expect(await withUploadLock(root, () => 'next operation')).toBe('next operation');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user