Add configurable history and storage retention policies

src/lib/server/game-files/index.ts

@@ -1,3 +1,4 @@
+import { fleetSettings } from '../settings.js';
 import { randomUUID } from 'node:crypto';
 import { mkdir, rename, stat, rm, lstat } from 'node:fs/promises';
 import { dirname, join, resolve } from 'node:path';
@@ -7,7 +8,7 @@ import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hos
 import { ensureGameFileSchema } from './schema.js';
 import { ensureHostSchema } from '../hosts/schema.js';
 import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js';
-import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock } from './uploads.js';
+import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock, uploadIsActive } from './uploads.js';
 import { GAME_FILE_GAMES } from '../../game-files.js';
 import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState, LibraryTransferProgress } from '../../game-files.js';
 
@@ -137,6 +138,7 @@ export function createLibraryUpload(input: Record<string, unknown>): GameLibrary
 type UploadEntry = { path: string; size: number; lastModified: number };
 function requireBrowserUpload(id: string): GameLibraryVersion {
     const version = getLibraryVersion(id);
+    if (database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(id)) throw Object.assign(new Error('Upload files expired under the retention policy.'), { status: 410 });
     if (!version || version.sourceHostId !== null || !database().prepare('SELECT 1 FROM game_file_upload_entries WHERE version_id=? LIMIT 1').get(id)) {
         throw Object.assign(new Error('Folder upload not found.'), { status: 404 });
     }
@@ -153,6 +155,7 @@ async function uploadStorageRoot(version: GameLibraryVersion): Promise<string> {
 }
 export async function browserUploadStatus(id: string) {
     const version = requireBrowserUpload(id);
+    touchUpload(id);
     const status = version.state === 'ready' ? 'published' as const : publishing.has(id) ? 'verifying' as const : version.state === 'failed' ? 'failed' as const : 'pending' as const;
     const root = await uploadStorageRoot(version);
     const entries = uploadEntries(id);
@@ -169,15 +172,18 @@ export async function uploadBrowserFile(id: string, path: string, chunk: { offse
         if (version.state === 'ready' || version.manifestSha256 || publishing.has(id)) throw Object.assign(new Error('This upload is no longer writable.'), { status: 409 });
     };
     checkWritable();
+    touchUpload(id);
     const file = database().prepare('SELECT path,size,last_modified AS lastModified FROM game_file_upload_entries WHERE version_id=? AND path=?').get(id, path) as UploadEntry | undefined;
     if (!file) throw new Error('File is not listed in this upload.');
     const progress = await receiveUploadChunk(stagingRoot(id), file, chunk, body, checkWritable);
     database().prepare("UPDATE game_file_versions SET state='importing',error=NULL WHERE id=? AND state IN ('importing','failed')").run(id);
+    touchUpload(id);
     return progress;
 }
 export async function publishBrowserUpload(id: string): Promise<GameLibraryVersion> {
     const version = requireBrowserUpload(id);
     if (version.state === 'ready') return version;
+    touchUpload(id);
     const existing = publishing.get(id); if (existing) return existing;
     const task = withUploadLock(stagingRoot(id), async () => {
         const root = await uploadStorageRoot(version);
@@ -241,6 +247,7 @@ export function createLibraryImport(input: Record<string, unknown>): GameLibrary
 
 export function requireImportOwner(hostId: string, id: string): GameLibraryVersion {
     const version = getLibraryVersion(id);
+    if (database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(id)) throw Object.assign(new Error('Upload files expired under the retention policy.'), { status: 410 });
     if (!version || version.sourceHostId !== hostId) throw new Error('Import not found for this host.');
     return version;
 }
@@ -473,7 +480,7 @@ export function reconcileHostLibraryCleanup(hostId: string): void {
     const host = getHost(hostId);
     if (!host?.connected || !host.credentialActive || host.mode === 'disabled' || host.decommission
         || !host.inventory.capabilities?.includes('library.remove')) return;
-    const latest = new Set(latestPublishedLibraries().map(version => version.id));
+    const latest = retainedGameFileIds();
     const caches = handle.prepare('SELECT * FROM host_library_caches WHERE host_id=?').all(hostId) as CacheRow[];
     for (const cache of caches) {
         if (latest.has(cache.version_id) || (cache.status === 'missing' && cache.bytes_downloaded === 0) || libraryInUse(cache.version_id, hostId)) continue;
@@ -493,7 +500,8 @@ export function cleanupControllerLibraries(): Promise<void> {
     const handle = database();
     const current = cleaning.get(handle); if (current) return current;
     const task = (async () => {
-        const latest = new Set(latestPublishedLibraries().map(version => version.id));
+        await cleanupAbandonedUploads();
+        const latest = retainedGameFileIds();
         const versions = handle.prepare(`SELECT v.id,v.storage_location,l.game FROM game_file_versions v
             JOIN game_file_libraries l ON l.id=v.library_id LEFT JOIN game_file_cleanup c ON c.version_id=v.id
             WHERE v.state='ready' AND c.removed_at IS NULL`).all() as { id: string; storage_location: string; game: string }[];
@@ -521,3 +529,67 @@ export function cleanupControllerLibraries(): Promise<void> {
     void task.finally(() => cleaning.delete(handle)).catch(() => {});
     return task;
 }
+
+function touchUpload(id: string) {
+    database().prepare("UPDATE game_file_versions SET last_activity_at=? WHERE id=? AND state!='ready'").run(new Date().toISOString(),id);
+}
+
+/** Latest N published versions are retained per game; grace starts at supersession. */
+export function retainedGameFileIds(settings = fleetSettings()): Set<string> {
+    const versions = database().prepare(`SELECT v.id,l.game,v.published_at FROM game_file_versions v
+        JOIN game_file_libraries l ON l.id=v.library_id WHERE v.state='ready'
+        ORDER BY v.published_at DESC,v.created_at DESC,v.rowid DESC`).all() as { id: string; game: string; published_at: string }[];
+    const kept = new Set<string>();
+    const seen = new Map<string,{ count: number; newerAt: string }>();
+    for (const v of versions) {
+        const prior = seen.get(v.game);
+        if (!prior || prior.count < settings.gameFileKeepCount
+            || Date.parse(prior.newerAt) > Date.now() - settings.gameFileGraceDays * 86400_000) kept.add(v.id);
+        seen.set(v.game,{ count: (prior?.count ?? 0) + 1, newerAt: v.published_at });
+    }
+    for (const row of database().prepare("SELECT record_id FROM retention_controls WHERE category='game-file' AND protected=1").all() as { record_id: string }[]) kept.add(row.record_id);
+    return kept;
+}
+
+function abandonedUploads(settings = fleetSettings()) {
+    const days = settings.uploadKeepDays;
+    if (!days) return [];
+    return database().prepare(`SELECT v.id,v.storage_location,l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
+        WHERE v.state IN ('importing','failed') AND v.source_host_id='' AND COALESCE(v.last_activity_at,v.created_at)<?
+        AND NOT EXISTS (SELECT 1 FROM game_file_cleanup c WHERE c.version_id=v.id AND c.removed_at IS NOT NULL)
+        AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='upload' AND c.record_id=v.id AND c.protected=1)
+        ORDER BY v.created_at LIMIT 100`).all(new Date(Date.now()-days*86400_000).toISOString()) as { id: string; storage_location: string; game: string }[];
+}
+async function cleanupAbandonedUploads() {
+    for (const upload of abandonedUploads()) {
+        if (publishing.has(upload.id) || uploadIsActive(stagingRoot(upload.id))) continue;
+        await withUploadLock(stagingRoot(upload.id), async () => {
+            // A chunk or resume request may have refreshed the timestamp before lock admission.
+            if (!abandonedUploads().some(row => row.id === upload.id)) return;
+            if (!/^[a-zA-Z0-9_-]+$/.test(upload.id)) return;
+            const expected = join(gameFilesRoot(),upload.game,upload.id);
+            if (resolve(upload.storage_location) !== expected) return;
+            database().prepare('INSERT OR IGNORE INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(upload.id,new Date().toISOString());
+            // Paths are controller-derived. Validate parents before recursive removal.
+            for (const path of [gameFilesRoot(),join(gameFilesRoot(),'.imports'),join(gameFilesRoot(),upload.game),expected,stagingRoot(upload.id),`${stagingRoot(upload.id)}.chunks`]) {
+                const info = await lstat(path).catch(cause => { if (cause.code === 'ENOENT') return null; throw cause; });
+                if (info?.isSymbolicLink()) throw new Error('Refusing upload cleanup through a symlink.');
+            }
+            for (const path of [expected,stagingRoot(upload.id),`${stagingRoot(upload.id)}.chunks`]) await rm(path,{ recursive: true,force: true });
+            database().prepare("UPDATE game_file_versions SET state='failed',error='Upload expired under the retention policy.' WHERE id=? AND state!='ready'").run(upload.id);
+            database().prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?').run(new Date().toISOString(),upload.id);
+        });
+    }
+}
+
+export function gameFileRetentionPreview(settings = fleetSettings()) {
+    const kept = retainedGameFileIds(settings);
+    const versions = database().prepare(`SELECT v.id,v.size_bytes FROM game_file_versions v LEFT JOIN game_file_cleanup c ON c.version_id=v.id
+        WHERE v.state='ready' AND c.removed_at IS NULL`).all() as { id: string; size_bytes: number }[];
+    const eligible = versions.filter(v => !kept.has(v.id) && !libraryInUse(v.id) && !database().prepare("SELECT 1 FROM host_library_caches WHERE version_id=? AND (status!='missing' OR bytes_downloaded>0)").get(v.id));
+    const uploads = abandonedUploads(settings).filter(v => !publishing.has(v.id) && !uploadIsActive(stagingRoot(v.id)));
+    return [
+        { category: 'game-file',label: 'Unused controller game files',eligible: eligible.length,payloadBytes: eligible.reduce((n,v)=>n+v.size_bytes,0),records: eligible.slice(0,20).map(v=>({ id: v.id })) },
+        { category: 'upload',label: 'Abandoned uploads',eligible: uploads.length,payloadBytes: 0,records: uploads.slice(0,20).map(v=>({ id: v.id })) }
+    ];
+}

src/lib/server/game-files/schema.ts

@@ -49,6 +49,8 @@ export function ensureGameFileSchema(handle: Database.Database): void {
         CREATE TRIGGER IF NOT EXISTS game_file_versions_immutable BEFORE UPDATE ON game_file_versions
         WHEN OLD.state = 'ready' BEGIN SELECT RAISE(ABORT, 'Published game-file versions are immutable'); END;
     `);
+    const versionColumns = handle.prepare('PRAGMA table_info(game_file_versions)').all() as { name: string }[];
+    if (!versionColumns.some(column => column.name === 'last_activity_at')) handle.exec('ALTER TABLE game_file_versions ADD COLUMN last_activity_at TEXT');
     const cacheColumns = handle.prepare('PRAGMA table_info(host_library_caches)').all() as { name: string }[];
     if (!cacheColumns.some(column => column.name === 'bytes_verified')) handle.exec('ALTER TABLE host_library_caches ADD COLUMN bytes_verified INTEGER');
     if (!cacheColumns.some(column => column.name === 'transfer_progress')) handle.exec('ALTER TABLE host_library_caches ADD COLUMN transfer_progress TEXT');

src/lib/server/game-files/uploads.ts

@@ -46,6 +46,7 @@ export function validateUploadPlan(input: unknown): UploadFilePlan[] {
 
 interface UploadLocks { barrier: Promise<void>; writers: number; files: Map<string, Promise<void>>; active: Set<Promise<void>> }
 const uploadLocks = new Map<string, UploadLocks>();
+export function uploadIsActive(root: string): boolean { return uploadLocks.has(resolve(root)); }
 function locksFor(key: string): UploadLocks {
     let locks = uploadLocks.get(key);
     if (!locks) { locks = { barrier: Promise.resolve(), writers: 0, files: new Map(), active: new Set() }; uploadLocks.set(key, locks); }