Automatically remove unused game-file versions

src/lib/game-files.ts

@@ -25,6 +25,8 @@ export interface GameLibraryVersion {
     createdAt: string; publishedAt: string | null; storageLocation: string;
     sourceHostId: string | null; sourceVolume: string | null; error: string | null; operationId: string | null;
     caches: HostLibraryCache[];
+    filesRemoved?: boolean;
+    filesRemoving?: boolean;
 }
 export interface GameLibrary {
     id: string; name: string; game: GameLibraryGame; description: string;

src/lib/server/game-files/http.ts

@@ -2,7 +2,7 @@ import { createReadStream } from 'node:fs';
 import { stat } from 'node:fs/promises';
 import { Readable } from 'node:stream';
 import { json } from '@sveltejs/kit';
-import { getLibraryFile } from './index.js';
+import { getLibraryFile, libraryFilesRetired } from './index.js';
 import { assertRegularPath, parseByteRange } from './storage.js';
 
 export function gameFileError(cause: unknown): Response {
@@ -28,6 +28,7 @@ export async function readGameJson(request: Request, maxBytes = 64 * 1024): Prom
 }
 
 export async function serveGameFile(request: Request, versionId: string, requestedPath: string): Promise<Response> {
+    if (libraryFilesRetired(versionId)) return json({ error: 'These outdated game files have been removed.' }, { status: 410 });
     const file = getLibraryFile(versionId, requestedPath);
     if (!file || file.state !== 'ready') return json({ error: 'File is not in a published manifest.' }, { status: 404 });
     const path = await assertRegularPath(file.storageLocation, file.path);

src/lib/server/game-files/index.ts

@@ -1,16 +1,17 @@
 import { randomUUID } from 'node:crypto';
-import { mkdir, rename, stat } from 'node:fs/promises';
+import { mkdir, rename, stat, rm, lstat } from 'node:fs/promises';
 import { dirname, join, resolve } from 'node:path';
 import { db } from '../db/index.js';
 import { config } from '../config.js';
 import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hosts/index.js';
 import { ensureGameFileSchema } from './schema.js';
+import { ensureHostSchema } from '../hosts/schema.js';
 import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js';
 import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock } from './uploads.js';
 import { GAME_FILE_GAMES } from '../../game-files.js';
 import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState, LibraryTransferProgress } from '../../game-files.js';
 
-function database() { const handle = db(); ensureGameFileSchema(handle); return handle; }
+function database() { const handle = db(); ensureGameFileSchema(handle); ensureHostSchema(handle); return handle; }
 export const gameFilesRoot = () => resolve(process.env.GAME_FILES_ROOT ?? join(dirname(config.dbPath), 'game-files'));
 const stagingRoot = (id: string) => join(gameFilesRoot(), '.imports', id);
 type VersionRow = { id: string; library_id: string; game: GameLibraryGame; version: string; description: string; dlc: string; state: GameLibraryVersion['state']; size_bytes: number; manifest: string; manifest_sha256: string | null; created_at: string; published_at: string | null; storage_location: string; source_host_id: string; source_volume: string; error: string | null; operation_id: string | null };
@@ -22,6 +23,8 @@ function versionFromRow(row: VersionRow & { file_count?: number }, summaryCaches
     const files = summaryCaches ? [] : JSON.parse(row.manifest) as GameFileEntry[];
     return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version,
         description: row.description, state: row.state, sizeBytes: row.size_bytes,
+        filesRemoved: !!database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=? AND removed_at IS NOT NULL').get(row.id),
+        filesRemoving: !!database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=? AND removed_at IS NULL').get(row.id),
         files, fileCount: row.file_count ?? (files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count), manifestSha256: row.manifest_sha256, createdAt: row.created_at,
         publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id || null,
         sourceVolume: row.source_volume || null, error: row.error, operationId: row.operation_id,
@@ -31,6 +34,9 @@ export function getLibraryVersion(id: string): GameLibraryVersion | null {
     const row = database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=?').get(id) as VersionRow | undefined;
     return row ? versionFromRow(row) : null;
 }
+export function libraryFilesRetired(versionId: string): boolean {
+    return !!database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(versionId);
+}
 export function getLibraryFile(versionId: string, path: string) {
     validateGamePath(path);
     return database().prepare(`SELECT f.path,f.size,f.sha256,v.state,v.source_host_id AS sourceHostId,v.storage_location AS storageLocation
@@ -75,7 +81,8 @@ export function getHostLibraryCache(hostId: string, versionId: string): HostLibr
 }
 export function requireReadyLibrary(hostId: string, versionId: string): GameLibraryVersion {
     const version = getLibraryVersion(versionId);
-    if (!version || version.state !== 'ready') throw new Error('Select a published game-file version.');
+    if (!version || version.state !== 'ready' || libraryFilesRetired(versionId)) throw new Error('Select an available published game-file version.');
+    if (database().prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.');
     const cache = getHostLibraryCache(hostId, versionId);
     if (!cache || !['ready', 'outdated'].includes(cache.status) || cache.manifestSha256 !== version.manifestSha256) throw new Error('The selected host must finish downloading and verifying this game-file version.');
     return version;
@@ -179,7 +186,7 @@ export async function publishBrowserUpload(id: string): Promise<GameLibraryVersi
         return getLibraryVersion(id)!;
     });
     publishing.set(id, task);
-    try { return await task; } finally { publishing.delete(id); }
+    try { const result = await task; await cleanupControllerLibraries(); return result; } finally { publishing.delete(id); }
 }
 export async function requestBrowserPublication(id: string) {
     const version = requireBrowserUpload(id);
@@ -267,7 +274,7 @@ export async function publishImport(hostId: string, id: string): Promise<GameLib
         return getLibraryVersion(id)!;
     })();
     publishing.set(id, task);
-    try { return await task; } finally { publishing.delete(id); }
+    try { const result = await task; await cleanupControllerLibraries(); return result; } finally { publishing.delete(id); }
 }
 
 /** Large libraries can take minutes to verify. Keep HTTP requests bounded while the source host polls. */
@@ -364,6 +371,8 @@ export function updateHostLibraryCaches(hostId: string, input: unknown, options:
 export function queueLibraryDownload(hostId: string, versionId: string, options: { automatic?: boolean } = {}) {
     const version = getLibraryVersion(versionId);
     if (!version || version.state !== 'ready') throw new Error('Choose a published library version.');
+    if (libraryFilesRetired(versionId)) throw new Error('These outdated game files have been removed. Upload them again to use them.');
+    if (database().prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.');
     const host = getHost(hostId);
     if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled host.');
     const prior = getHostLibraryCache(hostId, versionId);
@@ -410,3 +419,70 @@ export function libraryOverview() {
         })
     })) })), hosts };
 }
+
+/** Any live reservation (including stopped or provisioning instances) pins its files. */
+function libraryInUse(versionId: string, hostId?: string): boolean {
+    const handle = database();
+    // Do not infer an empty fleet if its schema is unavailable.
+    if (!handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_instances'").get()) return true;
+    if (handle.prepare(`SELECT 1 FROM native_instances WHERE library_id=? AND retired_at IS NULL ${hostId ? 'AND host_id=?' : ''} LIMIT 1`)
+        .get(versionId, ...(hostId ? [hostId] : []))) return true;
+    return !!handle.prepare(`SELECT 1 FROM ec_host_operations WHERE state IN ('pending','accepted','running','unknown')
+        AND (json_extract(payload,'$.libraryVersionId')=? OR json_extract(payload,'$.libraryId')=? OR json_extract(payload,'$.importId')=?)
+        ${hostId ? 'AND host_id=?' : ''} LIMIT 1`).get(versionId, versionId, versionId, ...(hostId ? [hostId] : []));
+}
+
+/** Queue serialized, recoverable host cleanup; an offline host keeps its copy pinned. */
+export function reconcileHostLibraryCleanup(hostId: string): void {
+    const handle = database();
+    const host = getHost(hostId);
+    if (!host?.connected || !host.credentialActive || host.mode === 'disabled' || host.decommission
+        || !host.inventory.capabilities?.includes('library.remove')) return;
+    const latest = new Set(latestPublishedLibraries().map(version => version.id));
+    const caches = handle.prepare('SELECT * FROM host_library_caches WHERE host_id=?').all(hostId) as CacheRow[];
+    for (const cache of caches) {
+        if (latest.has(cache.version_id) || (cache.status === 'missing' && cache.bytes_downloaded === 0) || libraryInUse(cache.version_id, hostId)) continue;
+        const previous = handle.prepare(`SELECT state,updated_at FROM ec_host_operations WHERE host_id=? AND kind='library.remove'
+            AND json_extract(payload,'$.libraryVersionId')=? ORDER BY rowid DESC LIMIT 1`).get(hostId, cache.version_id) as { state: string; updated_at: string } | undefined;
+        // Wait for fresh inventory after completion, and back off failures.
+        if (previous && (Date.parse(cache.updated_at) <= Date.parse(previous.updated_at)
+            || previous.state === 'failed' && Date.now() - Date.parse(previous.updated_at) < 60_000)) continue;
+        const version = getLibraryVersion(cache.version_id)!;
+        queueHostOperation(hostId, 'library.remove', { libraryVersionId: version.id, game: version.game });
+    }
+}
+
+const cleaning = new WeakMap<ReturnType<typeof database>, Promise<void>>();
+/** Reserve deletion durably before yielding, so new downloads cannot race filesystem removal. */
+export function cleanupControllerLibraries(): Promise<void> {
+    const handle = database();
+    const current = cleaning.get(handle); if (current) return current;
+    const task = (async () => {
+        const latest = new Set(latestPublishedLibraries().map(version => version.id));
+        const versions = handle.prepare(`SELECT v.id,v.storage_location,l.game FROM game_file_versions v
+            JOIN game_file_libraries l ON l.id=v.library_id LEFT JOIN game_file_cleanup c ON c.version_id=v.id
+            WHERE v.state='ready' AND c.removed_at IS NULL`).all() as { id: string; storage_location: string; game: string }[];
+        for (const version of versions) {
+            if (latest.has(version.id) || libraryInUse(version.id)) continue;
+            // Missing partial downloads still occupy disk. Require a full absence report from every host.
+            if (handle.prepare("SELECT 1 FROM host_library_caches WHERE version_id=? AND (status!='missing' OR bytes_downloaded>0) LIMIT 1").get(version.id)) continue;
+            // Only delete the controller-owned version directory, never an arbitrary stored path.
+            const expected = join(gameFilesRoot(), version.game, version.id);
+            if (!/^[a-zA-Z0-9_-]+$/.test(version.id) || resolve(version.storage_location) !== expected) continue;
+            handle.prepare('INSERT OR IGNORE INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(version.id, new Date().toISOString());
+            try {
+                for (const path of [gameFilesRoot(), dirname(expected), expected]) {
+                    const info = await lstat(path).catch(cause => { if (cause.code === 'ENOENT') return null; throw cause; });
+                    if (info?.isSymbolicLink()) throw new Error('Refusing game-file cleanup through a symlink.');
+                }
+                await rm(expected, { recursive: true, force: true });
+                handle.prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?').run(new Date().toISOString(), version.id);
+            } catch (cause) {
+                console.error(`Game-file cleanup will retry for ${version.id}:`, cause);
+            }
+        }
+    })();
+    cleaning.set(handle, task);
+    void task.finally(() => cleaning.delete(handle)).catch(() => {});
+    return task;
+}

src/lib/server/game-files/schema.ts

@@ -34,6 +34,10 @@ export function ensureGameFileSchema(handle: Database.Database): void {
             size INTEGER NOT NULL, last_modified INTEGER NOT NULL,
             PRIMARY KEY(version_id,path)
         );
+        CREATE TABLE IF NOT EXISTS game_file_cleanup (
+            version_id TEXT PRIMARY KEY REFERENCES game_file_versions(id),
+            started_at TEXT NOT NULL, removed_at TEXT
+        );
         INSERT OR IGNORE INTO game_file_entries(version_id,path,size,sha256)
             SELECT v.id,json_extract(f.value,'$.path'),json_extract(f.value,'$.size'),json_extract(f.value,'$.sha256')
             FROM game_file_versions v,json_each(v.manifest) f;

src/lib/server/pipeline/worker.ts

@@ -16,6 +16,7 @@ import { STATUS_REFRESH_MS } from '../../server-freshness.js';
 import { recordMetrics } from '../fleet/metrics.js';
 import { selectedZombieMap } from '../../map-aliases.js';
 import { runMaintenance } from '../maintenance/index.js';
+import { cleanupControllerLibraries } from '../game-files/index.js';
 import { trackActivity } from './activity.js';
 
 const TICK_MS = 15_000;
@@ -50,13 +51,14 @@ function reconcileRequesters(): void {
     const requesters: [string, string, () => Promise<unknown>][] = [
         ['campaign', 'Reconciling campaigns', sweep],
         ['beta', 'Reconciling beta tests', betaTick],
-        ['maintenance', 'Checking maintenance and recovery', runMaintenance]
+        ['maintenance', 'Checking maintenance and recovery', runMaintenance],
+        ['game-files', 'Removing unused game files', cleanupControllerLibraries]
     ];
     for (const [key, label, work] of requesters) {
         if (requesterFlights.has(key)) continue;
         const flight = trackActivity(key, label, work)
             .catch(err => { console.error(`[worker] ${key} error:`, err); })
-            .then(async () => { await tick(true); })
+            .then(async () => { if (key !== 'game-files') await tick(true); })
             .finally(() => { requesterFlights.delete(key); });
         requesterFlights.set(key, flight);
     }