Support resumable game folder uploads and remove DLC declarations

src/lib/server/game-files/http.ts

@@ -7,7 +7,8 @@ import { assertRegularPath, parseByteRange } from './storage.js';
 
 export function gameFileError(cause: unknown): Response {
     const status = typeof cause === 'object' && cause !== null && 'status' in cause && typeof cause.status === 'number' ? cause.status : 400;
-    return json({ error: cause instanceof Error ? cause.message : 'Game file request failed.' }, { status });
+    const receivedBytes = typeof cause === 'object' && cause !== null && 'receivedBytes' in cause && Number.isSafeInteger(cause.receivedBytes) ? cause.receivedBytes : undefined;
+    return json({ error: cause instanceof Error ? cause.message : 'Game file request failed.', ...(receivedBytes === undefined ? {} : { receivedBytes }) }, { status });
 }
 export async function readGameJson(request: Request, maxBytes = 64 * 1024): Promise<Record<string, unknown>> {
     if (!request.body) throw new Error('A JSON request body is required.');

src/lib/server/game-files/index.ts

@@ -6,6 +6,7 @@ import { config } from '../config.js';
 import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hosts/index.js';
 import { ensureGameFileSchema } from './schema.js';
 import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js';
+import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock } from './uploads.js';
 import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState } from '../../game-files.js';
 
 function database() { const handle = db(); ensureGameFileSchema(handle); return handle; }
@@ -17,10 +18,10 @@ const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ hostId: row.host_id
 function versionFromRow(row: VersionRow): GameLibraryVersion {
     const files = JSON.parse(row.manifest) as GameFileEntry[];
     return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version,
-        description: row.description, dlc: JSON.parse(row.dlc), state: row.state, sizeBytes: row.size_bytes,
-        files, fileCount: files.length, manifestSha256: row.manifest_sha256, createdAt: row.created_at,
-        publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id,
-        sourceVolume: row.source_volume, error: row.error, operationId: row.operation_id,
+        description: row.description, state: row.state, sizeBytes: row.size_bytes,
+        files, fileCount: files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count, manifestSha256: row.manifest_sha256, createdAt: row.created_at,
+        publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id || null,
+        sourceVolume: row.source_volume || null, error: row.error, operationId: row.operation_id,
         caches: (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) };
 }
 export function getLibraryVersion(id: string): GameLibraryVersion | null {
@@ -60,6 +61,112 @@ function textField(input: unknown, label: string, max: number, required = true):
     if (typeof input !== 'string' || input.length > max || (required && !input.trim())) throw new Error(`Provide a valid ${label}.`);
     return input.trim();
 }
+
+/** Register a folder on the controller; no host or Docker volume is needed. */
+export function createLibraryUpload(input: Record<string, unknown>): GameLibraryVersion {
+    const game = input.game;
+    if (game !== 't5' && game !== 't6') throw new Error('Choose T5 or T6 game files.');
+    const name = textField(input.name, 'library name', 160);
+    const description = textField(input.description ?? '', 'description', 4000, false);
+    const version = textField(input.version, 'version label', 120);
+    const files = validateUploadPlan(input.files);
+    const handle = database();
+    const id = randomUUID(); const now = new Date().toISOString();
+    handle.transaction(() => {
+        let library = handle.prepare('SELECT id,game FROM game_file_libraries WHERE name=?').get(name) as { id: string; game: string } | undefined;
+        if (library && library.game !== game) throw new Error('This library name is already used for another game.');
+        if (!library) {
+            library = { id: randomUUID(), game };
+            handle.prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)').run(library.id, name, game, description, now);
+        }
+        if (handle.prepare('SELECT 1 FROM game_file_versions WHERE library_id=? AND version=?').get(library.id, version)) throw new Error('This version label already exists. Choose a new version.');
+        // Keep legacy non-null columns for existing databases and published versions.
+        handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,size_bytes,created_at,storage_location,source_host_id,source_volume)
+            VALUES(?,?,?,?,'[]','importing',?,?,?,'','')`).run(id, library.id, version, description, files.reduce((sum, file) => sum + file.size, 0), now, join(gameFilesRoot(), game, id));
+        const insert = handle.prepare('INSERT INTO game_file_upload_entries(version_id,path,size,last_modified) VALUES(?,?,?,?)');
+        for (const file of files) insert.run(id, file.path, file.size, file.lastModified);
+    })();
+    return getLibraryVersion(id)!;
+}
+
+type UploadEntry = { path: string; size: number; lastModified: number };
+function requireBrowserUpload(id: string): GameLibraryVersion {
+    const version = getLibraryVersion(id);
+    if (!version || version.sourceHostId !== null || !database().prepare('SELECT 1 FROM game_file_upload_entries WHERE version_id=? LIMIT 1').get(id)) {
+        throw Object.assign(new Error('Folder upload not found.'), { status: 404 });
+    }
+    return version;
+}
+function uploadEntries(id: string): UploadEntry[] {
+    return database().prepare('SELECT path,size,last_modified AS lastModified FROM game_file_upload_entries WHERE version_id=? ORDER BY path').all(id) as UploadEntry[];
+}
+async function uploadStorageRoot(version: GameLibraryVersion): Promise<string> {
+    // Recover publication interrupted after the atomic rename but before its DB commit.
+    try { if ((await stat(version.storageLocation)).isDirectory()) return version.storageLocation; }
+    catch (cause) { if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause; }
+    return stagingRoot(version.id);
+}
+export async function browserUploadStatus(id: string) {
+    const version = requireBrowserUpload(id);
+    const status = version.state === 'ready' ? 'published' as const : publishing.has(id) ? 'verifying' as const : version.state === 'failed' ? 'failed' as const : 'pending' as const;
+    const root = await uploadStorageRoot(version);
+    const entries = uploadEntries(id);
+    const files = [];
+    // Avoid opening thousands of files concurrently for a large folder.
+    for (const file of entries) files.push({ ...file, ...(status === 'published' ? { receivedBytes: file.size, complete: true } : await getUploadFileProgress(root, file)) });
+    const { files: _manifest, ...summary } = version;
+    return { status, chunkSize: MAX_UPLOAD_CHUNK_BYTES, files, version: summary, error: version.error };
+}
+export async function uploadBrowserFile(id: string, path: string, chunk: { offset: number; size: number; sha256: string }, body: ReadableStream<Uint8Array> | null) {
+    validateGamePath(path);
+    const checkWritable = () => {
+        const version = requireBrowserUpload(id);
+        if (version.state === 'ready' || version.manifestSha256 || publishing.has(id)) throw Object.assign(new Error('This upload is no longer writable.'), { status: 409 });
+    };
+    checkWritable();
+    const file = database().prepare('SELECT path,size,last_modified AS lastModified FROM game_file_upload_entries WHERE version_id=? AND path=?').get(id, path) as UploadEntry | undefined;
+    if (!file) throw new Error('File is not listed in this upload.');
+    const progress = await receiveUploadChunk(stagingRoot(id), file, chunk, body, checkWritable);
+    database().prepare("UPDATE game_file_versions SET state='importing',error=NULL WHERE id=? AND state IN ('importing','failed')").run(id);
+    return progress;
+}
+export async function publishBrowserUpload(id: string): Promise<GameLibraryVersion> {
+    const version = requireBrowserUpload(id);
+    if (version.state === 'ready') return version;
+    const existing = publishing.get(id); if (existing) return existing;
+    const task = withUploadLock(stagingRoot(id), async () => {
+        const root = await uploadStorageRoot(version);
+        const files = await hashUploadedFiles(root, uploadEntries(id));
+        const checksum = manifestChecksum(files);
+        if (version.manifestSha256 && checksum !== version.manifestSha256) throw new Error('Stored files no longer match the verified manifest.');
+        database().transaction(() => {
+            database().prepare("UPDATE game_file_versions SET state='importing',manifest=?,manifest_sha256=?,error=NULL WHERE id=? AND state IN ('importing','failed')").run(JSON.stringify(files), checksum, id);
+            database().prepare('DELETE FROM game_file_entries WHERE version_id=?').run(id);
+            const insert = database().prepare('INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)');
+            for (const file of files) insert.run(id, file.path, file.size, file.sha256);
+        })();
+        if (root !== version.storageLocation) {
+            await mkdir(dirname(version.storageLocation), { recursive: true });
+            await rename(root, version.storageLocation);
+        }
+        database().prepare("UPDATE game_file_versions SET state='ready',published_at=?,error=NULL WHERE id=? AND state='importing'").run(new Date().toISOString(), id);
+        return getLibraryVersion(id)!;
+    });
+    publishing.set(id, task);
+    try { return await task; } finally { publishing.delete(id); }
+}
+export async function requestBrowserPublication(id: string) {
+    const version = requireBrowserUpload(id);
+    if (version.state !== 'ready' && !publishing.has(id)) {
+        database().prepare("UPDATE game_file_versions SET state='importing',error=NULL WHERE id=? AND state IN ('importing','failed')").run(id);
+        void publishBrowserUpload(id).catch(cause => {
+            database().prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'")
+                .run((cause instanceof Error ? cause.message : 'Folder publication failed.').slice(0, 2000), id);
+        });
+    }
+    return browserUploadStatus(id);
+}
+
 export function createLibraryImport(input: Record<string, unknown>): GameLibraryVersion {
     const game = input.game;
     if (game !== 't5' && game !== 't6') throw new Error('Choose T5 or T6 game files.');
@@ -69,9 +176,6 @@ export function createLibraryImport(input: Record<string, unknown>): GameLibrary
     const name = textField(input.name, 'library name', 160);
     const description = textField(input.description ?? '', 'description', 4000, false);
     const version = textField(input.version, 'version label', 120);
-    const dlcInput = input.dlc ?? [];
-    if (!Array.isArray(dlcInput) || dlcInput.length > 64) throw new Error('DLC must be a list of content names.');
-    const dlc = dlcInput.map(value => textField(value, 'DLC name', 120));
     const sourceVolume = game === 't5' ? 't5_game_files' : 't6_game_files';
     const handle = database();
     let library = handle.prepare('SELECT id,game FROM game_file_libraries WHERE name=?').get(name) as { id: string; game: string } | undefined;
@@ -84,7 +188,7 @@ export function createLibraryImport(input: Record<string, unknown>): GameLibrary
         }
         if (handle.prepare('SELECT 1 FROM game_file_versions WHERE library_id=? AND version=?').get(library.id, version)) throw new Error('This version label already exists. Choose a new version.');
         handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume)
-            VALUES(?,?,?,?,?,'importing',?,?,?,?)`).run(id, library.id, version, description, JSON.stringify(dlc), now, join(gameFilesRoot(), game, id), hostId, sourceVolume);
+            VALUES(?,?,?,?,?,'importing',?,?,?,?)`).run(id, library.id, version, description, '[]', now, join(gameFilesRoot(), game, id), hostId, sourceVolume);
     })();
     try {
         const operation = queueHostOperation(hostId, 'library.import', {

src/lib/server/game-files/schema.ts

@@ -29,6 +29,11 @@ export function ensureGameFileSchema(handle: Database.Database): void {
             version_id TEXT NOT NULL REFERENCES game_file_versions(id), path TEXT NOT NULL,
             size INTEGER NOT NULL, sha256 TEXT NOT NULL, PRIMARY KEY(version_id,path)
         );
+        CREATE TABLE IF NOT EXISTS game_file_upload_entries (
+            version_id TEXT NOT NULL REFERENCES game_file_versions(id), path TEXT NOT NULL,
+            size INTEGER NOT NULL, last_modified INTEGER NOT NULL,
+            PRIMARY KEY(version_id,path)
+        );
         INSERT OR IGNORE INTO game_file_entries(version_id,path,size,sha256)
             SELECT v.id,json_extract(f.value,'$.path'),json_extract(f.value,'$.size'),json_extract(f.value,'$.sha256')
             FROM game_file_versions v,json_each(v.manifest) f;

src/lib/server/game-files/storage.ts

@@ -63,13 +63,18 @@ export async function assertRegularPath(root: string, path: string, allowMissing
     return result;
 }
 
-export async function verifyGameFile(root: string, file: GameFileEntry): Promise<void> {
+/** Compute a file digest without buffering game archives in memory. */
+export async function hashGameFile(root: string, file: Pick<GameFileEntry, 'path' | 'size'>): Promise<string> {
     const path = await assertRegularPath(root, file.path);
     const info = await stat(path);
     if (info.size !== file.size) throw new Error(`Size verification failed for ${file.path}.`);
     const hash = createHash('sha256');
     for await (const chunk of createReadStream(path)) hash.update(chunk);
-    if (hash.digest('hex') !== file.sha256) throw new Error(`Checksum verification failed for ${file.path}.`);
+    return hash.digest('hex');
+}
+
+export async function verifyGameFile(root: string, file: GameFileEntry): Promise<void> {
+    if (await hashGameFile(root, file) !== file.sha256) throw new Error(`Checksum verification failed for ${file.path}.`);
 }
 
 /** Stream to a private temporary file; publish only after byte count and SHA256 match. */

src/lib/server/game-files/uploads.ts

@@ -0,0 +1,155 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants, createReadStream, createWriteStream } from 'node:fs';
+import { lstat, mkdir, open, rm } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+import { Readable, Transform } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import type { GameFileEntry } from '../../game-files.js';
+import { assertRegularPath, hashGameFile, validateManifest } from './storage.js';
+
+export const MAX_UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024;
+
+export interface UploadFilePlan {
+    path: string;
+    size: number;
+    lastModified: number;
+}
+
+export interface UploadFileProgress {
+    receivedBytes: number;
+    complete: boolean;
+}
+
+export class UploadOffsetError extends Error {
+    readonly status = 409;
+    readonly receivedBytes: number;
+    constructor(receivedBytes: number) {
+        super(`Upload offset does not match the stored file. Resume at byte ${receivedBytes}.`);
+        this.name = 'UploadOffsetError';
+        this.receivedBytes = receivedBytes;
+    }
+}
+
+/** Apply the same path, case, size and file/directory collision rules as published manifests. */
+export function validateUploadPlan(input: unknown): UploadFilePlan[] {
+    if (!Array.isArray(input) || !input.length || input.length > 200_000) throw new Error('Provide a folder containing 1–200000 files.');
+    const modified = new Map<string, number>();
+    const manifest = validateManifest(input.map(raw => {
+        if (!raw || typeof raw !== 'object') throw new Error('Invalid upload file.');
+        const lastModified = raw.lastModified === undefined ? 0 : raw.lastModified;
+        if (!Number.isSafeInteger(lastModified) || lastModified < 0) throw new Error('Invalid file modification time.');
+        modified.set(raw.path, lastModified);
+        return { path: raw.path, size: raw.size, sha256: '0'.repeat(64) };
+    }));
+    return manifest.map(file => ({ path: file.path, size: file.size, lastModified: modified.get(file.path)! }));
+}
+
+const uploadLocks = new Map<string, Promise<void>>();
+
+/** Serialize transfers and final publication for one import; a rejected operation never poisons the lock. */
+export async function withUploadLock<T>(root: string, callback: () => Promise<T> | T): Promise<T> {
+    const key = resolve(root);
+    const previous = uploadLocks.get(key) ?? Promise.resolve();
+    let release!: () => void;
+    const current = new Promise<void>(done => { release = done; });
+    uploadLocks.set(key, current);
+    await previous;
+    try { return await callback(); }
+    finally {
+        release();
+        if (uploadLocks.get(key) === current) uploadLocks.delete(key);
+    }
+}
+
+/** File size is the resume checkpoint, including after a controller restart during an append. */
+export async function getUploadFileProgress(root: string, file: Pick<UploadFilePlan, 'path' | 'size'>): Promise<UploadFileProgress> {
+    const path = await assertRegularPath(root, file.path, true);
+    try {
+        const info = await lstat(path);
+        if (!info.isFile() || info.isSymbolicLink()) throw new Error('Game storage contains a non-regular path.');
+        if (info.size > file.size) throw new Error(`Stored upload exceeds the declared size of ${file.path}.`);
+        return { receivedBytes: info.size, complete: info.size === file.size };
+    } catch (cause) {
+        if ((cause as NodeJS.ErrnoException).code === 'ENOENT') return { receivedBytes: 0, complete: false };
+        throw cause;
+    }
+}
+
+/**
+ * Verify one bounded request before touching the destination. Only verified bytes may be
+ * appended, so a process interruption during the append can resume at the actual file size.
+ * beforeWrite runs under the lock: callers must recheck that the import is still writable.
+ */
+export async function receiveUploadChunk(
+    root: string,
+    file: Pick<UploadFilePlan, 'path' | 'size'>,
+    chunk: { offset: number; size: number; sha256: string },
+    body: ReadableStream<Uint8Array> | null,
+    beforeWrite?: () => Promise<void> | void
+): Promise<UploadFileProgress> {
+    if (!Number.isSafeInteger(chunk.offset) || chunk.offset < 0 || !Number.isSafeInteger(chunk.size)
+        || chunk.size < 0 || chunk.size > MAX_UPLOAD_CHUNK_BYTES || chunk.offset + chunk.size > file.size
+        || !Number.isSafeInteger(chunk.offset + chunk.size) || !/^[a-f0-9]{64}$/.test(chunk.sha256)
+        || (chunk.size === 0 && file.size !== 0)) {
+        throw new Error('Invalid upload chunk size, offset or SHA256.');
+    }
+    if (!body && chunk.size !== 0) throw new Error('A file chunk body is required.');
+
+    return withUploadLock(root, async () => {
+        await beforeWrite?.();
+        const progress = await getUploadFileProgress(root, file);
+        if (chunk.offset !== progress.receivedBytes) throw new UploadOffsetError(progress.receivedBytes);
+        const destination = await assertRegularPath(root, file.path, true);
+        // Keep transient chunks outside the tree that will become the immutable library.
+        const chunkRoot = `${resolve(root)}.chunks`;
+        const temporary = await assertRegularPath(chunkRoot, randomUUID(), true);
+        await mkdir(dirname(temporary), { recursive: true });
+        let received = 0;
+        const hash = createHash('sha256');
+        const verifier = new Transform({ transform(data, _encoding, done) {
+            received += data.length;
+            if (received > chunk.size) return done(new Error('Upload exceeds the declared chunk size.'));
+            hash.update(data); done(null, data);
+        } });
+        try {
+            await pipeline(body ? Readable.fromWeb(body as never) : Readable.from([]), verifier,
+                createWriteStream(temporary, { flags: 'wx', mode: 0o600, flush: true }));
+            if (received !== chunk.size || hash.digest('hex') !== chunk.sha256) throw new Error('Upload chunk checksum or size mismatch.');
+
+            await mkdir(dirname(destination), { recursive: true });
+            await assertRegularPath(root, file.path, true);
+            const target = await open(destination, constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW, 0o600);
+            try {
+                const info = await target.stat();
+                if (!info.isFile()) throw new Error('Game storage contains a non-regular path.');
+                if (info.size !== chunk.offset) throw new UploadOffsetError(info.size);
+                let position = chunk.offset;
+                try {
+                    for await (const data of createReadStream(temporary)) {
+                        const bytes = data as Buffer;
+                        let written = 0;
+                        while (written < bytes.length) {
+                            const result = await target.write(bytes, written, bytes.length - written, position);
+                            if (result.bytesWritten === 0) throw new Error('Unable to append the upload chunk.');
+                            written += result.bytesWritten;
+                            position += result.bytesWritten;
+                        }
+                    }
+                    await target.sync();
+                } catch (cause) {
+                    await target.truncate(chunk.offset);
+                    await target.sync();
+                    throw cause;
+                }
+            } finally { await target.close(); }
+            return { receivedBytes: chunk.offset + chunk.size, complete: chunk.offset + chunk.size === file.size };
+        } finally { await rm(temporary, { force: true }); }
+    });
+}
+
+/** Caller holds withUploadLock through hashing and publication to prevent concurrent writes. */
+export async function hashUploadedFiles(root: string, files: ReadonlyArray<Pick<UploadFilePlan, 'path' | 'size'>>): Promise<GameFileEntry[]> {
+    const manifest: GameFileEntry[] = [];
+    for (const file of files) manifest.push({ path: file.path, size: file.size, sha256: await hashGameFile(root, file) });
+    return validateManifest(manifest);
+}