Automatically remove unused game-file versions

agent/ec_agent.py

@@ -32,7 +32,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.6"
+VERSION = "1.4.7"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 LIBRARY_DOWNLOAD_WORKERS = 4
@@ -1094,7 +1094,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["instance.cancel-create", "operations.read-lane", "operations.health-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "instance.cancel-create", "operations.read-lane", "operations.health-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1722,6 +1722,35 @@ class Agent:
             self.publish_library_progress(payload, timing)
             raise
 
+    def library_remove(self, payload):
+        library_id = identifier(payload.get("libraryVersionId"))
+        directory = self.library_dir(payload.get("game"), library_id)
+        staging = directory.with_name("." + library_id + ".partial")
+        # Never follow substituted storage parents outside the managed tree.
+        for path in (self.root / "libraries", directory.parent, directory, staging):
+            if path.is_symlink():
+                raise Rejected("Refusing library cleanup through a symlink")
+        # Include stopped and unmanaged containers: any bind mount pins the files.
+        # Docker failures must stop cleanup, never masquerade as an empty host.
+        for container_id in self.docker.call("container", "ls", "--all", "--quiet").splitlines():
+            container = self.docker.inspect(container_id)
+            if container is None:
+                raise Rejected("Container inventory changed during library cleanup")
+            for mount in container["Mounts"]:
+                if not mount.get("Source"):  # tmpfs has no host filesystem source.
+                    continue
+                source = Path(mount["Source"]).resolve()
+                for target in (directory.resolve(), staging.resolve()):
+                    if source == target or target in source.parents or source in target.parents:
+                        raise Rejected("Game files are still mounted by a container")
+        self.checkpoint({"phase": "removing-library"})
+        for path in (directory, staging):
+            if path.exists():
+                shutil.rmtree(path)
+        self.library_progress.pop(library_id, None)
+        self.library_transfers.pop(library_id, None)
+        return {"libraryVersionId": library_id, "removed": True}
+
     def library_import(self, payload):
         game = payload.get("game")
         volume = payload.get("sourceVolume")
@@ -1803,6 +1832,8 @@ class Agent:
             return self.inventory()
         if kind == "library.download":
             return self.library_download(payload)
+        if kind == "library.remove":
+            return self.library_remove(payload)
         if kind == "library.import":
             return self.library_import(payload)
         if kind == "library.inspect":
@@ -1923,6 +1954,8 @@ class Agent:
                     return self.journal.update(operation["id"], "unknown", error="Import reconciliation is waiting for controller contact")
                 finally:
                     self.executing_id = None
+            if kind == "library.remove":
+                return self.journal.update(operation["id"], "completed", self.library_remove(payload))
             if kind == "library.download":
                 # This is a resumable immutable transfer, not a repeated
                 # container command. Full files skip transfer; verification

agent/tests/test_agent.py

@@ -74,6 +74,56 @@ class AgentTests(unittest.TestCase):
         self.agent.journal.close()
         self.temp.cleanup()
 
+    def test_library_remove_preserves_other_versions_and_is_idempotent(self):
+        other = self.agent.library_dir("t5", "other-library")
+        other.mkdir()
+        staging = self.library.with_name("." + self.library_id + ".partial")
+        staging.mkdir()
+        (staging / "partial").write_bytes(b"partial")
+        payload = {"libraryVersionId": self.library_id, "game": "t5"}
+        self.assertTrue(self.agent.library_remove(payload)["removed"])
+        self.assertFalse(self.library.exists())
+        self.assertFalse(staging.exists())
+        self.assertTrue(other.exists())
+        self.assertTrue(self.agent.library_remove(payload)["removed"])
+
+    def test_library_remove_refuses_stopped_container_mounts_and_docker_errors(self):
+        payload = {"libraryVersionId": self.library_id, "game": "t5"}
+        with patch.object(self.docker, "call", return_value="stopped-container"), patch.object(self.docker, "inspect", return_value={
+            "State": {"Running": False}, "Mounts": [{"Source": str(self.library / "content")}]
+        }):
+            with self.assertRaisesRegex(AgentError, "still mounted"):
+                self.agent.library_remove(payload)
+        with patch.object(self.docker, "call", side_effect=AgentError("Docker unavailable")):
+            with self.assertRaises(AgentError):
+                self.agent.library_remove(payload)
+        self.assertTrue((self.library / "content" / "game.ff").exists())
+
+    def test_library_remove_recovers_after_interrupted_deletion(self):
+        payload = {"libraryVersionId": self.library_id, "game": "t5"}
+        operation = {"id": "library-cleanup", "kind": "library.remove", "payload": payload}
+        self.agent.journal.accept(operation)
+        self.agent.journal.update(operation["id"], "running")
+        self.agent.journal.checkpoint(operation["id"], {"phase": "removing-library"})
+        (self.library / "manifest.json").unlink()
+        row = self.agent.reconcile(operation, self.agent.journal.get(operation["id"]))
+        self.assertEqual(row["state"], "completed")
+        self.assertFalse(self.library.exists())
+
+    def test_library_remove_ignores_unrelated_and_tmpfs_mounts(self):
+        with patch.object(self.docker, "call", return_value="other-container"), patch.object(self.docker, "inspect", return_value={
+            "Mounts": [{"Type": "tmpfs", "Source": ""}, {"Source": str(self.agent.root / "instances")}]
+        }):
+            self.agent.library_remove({"libraryVersionId": self.library_id, "game": "t5"})
+        self.assertFalse(self.library.exists())
+
+    def test_library_remove_refuses_symlink_storage(self):
+        linked = self.agent.library_dir("t5", "linked-library")
+        linked.symlink_to(self.library, target_is_directory=True)
+        with self.assertRaisesRegex(AgentError, "symlink"):
+            self.agent.library_remove({"libraryVersionId": "linked-library", "game": "t5"})
+        self.assertTrue(self.library.exists())
+
     def cpu_sample(self, line):
         with patch.object(Path, "open", mock_open(read_data=line)):
             return self.agent.cpu_usage_percent()

docs/native-hosts.md

@@ -408,8 +408,20 @@ automatic updates install it between host operations.
 Phase transitions are reported promptly and periodic heartbeats carry progress;
 controller round trips do not stall every transferred chunk.
 Corrupt or incomplete content never becomes Ready. Cache inventory is reconciled
-on heartbeats, including vanished caches. Previous immutable uploads remain usable
-when their exact manifest is verified.
+on heartbeats, including vanished caches. The latest published version of each game
+is always retained. With agent 1.4.7 or newer, an older host copy is automatically
+removed once no live instance reservation on that host references it. Stopped,
+provisioning, and rebuilding instances still pin their version. The agent also
+refuses deletion if any Docker container (including stopped containers) mounts
+those files. Cleanup includes partial downloads and retries after interruption.
+
+The controller removes an older version's files after every host has reported its
+copy absent and no instance or unresolved operation references that version.
+Offline hosts keep their last reported copies pinned until they reconnect.
+Upload metadata, manifests, and operation history remain visible; removed files
+cannot be downloaded again without a new upload. Cleanup runs on heartbeats,
+successful publication, and the controller background worker, without needing
+an open Game Files page.
 
 The controller and reverse proxy must accept 8 MiB upload requests and folder
 manifests up to 32 MiB. Compose sets `BODY_SIZE_LIMIT=Infinity`; application routes
@@ -418,8 +430,9 @@ limit of at least 32 MiB (for example `client_max_body_size 32m` in nginx).
 Existing uploads, host caches and server references remain usable. Previously
 named libraries are grouped under T5 or T6, and the most recently published
 upload for each game becomes its latest files. Their immutable IDs and stored
-content are preserved; no reupload is needed. Legacy source-host imports retain
-their existing IDs and can complete through their original workflow.
+content still in use are preserved; unused older files are cleaned up automatically.
+Legacy source-host imports retain their existing IDs and can complete through
+their original workflow.
 
 Reservations include host ID, local bind address, protocol and port. Wildcard
 binds conflict with concrete addresses on the same host. UDP and TCP reservations

src/lib/game-files.ts

@@ -25,6 +25,8 @@ export interface GameLibraryVersion {
     createdAt: string; publishedAt: string | null; storageLocation: string;
     sourceHostId: string | null; sourceVolume: string | null; error: string | null; operationId: string | null;
     caches: HostLibraryCache[];
+    filesRemoved?: boolean;
+    filesRemoving?: boolean;
 }
 export interface GameLibrary {
     id: string; name: string; game: GameLibraryGame; description: string;

src/lib/server/game-files/http.ts

@@ -2,7 +2,7 @@ import { createReadStream } from 'node:fs';
 import { stat } from 'node:fs/promises';
 import { Readable } from 'node:stream';
 import { json } from '@sveltejs/kit';
-import { getLibraryFile } from './index.js';
+import { getLibraryFile, libraryFilesRetired } from './index.js';
 import { assertRegularPath, parseByteRange } from './storage.js';
 
 export function gameFileError(cause: unknown): Response {
@@ -28,6 +28,7 @@ export async function readGameJson(request: Request, maxBytes = 64 * 1024): Prom
 }
 
 export async function serveGameFile(request: Request, versionId: string, requestedPath: string): Promise<Response> {
+    if (libraryFilesRetired(versionId)) return json({ error: 'These outdated game files have been removed.' }, { status: 410 });
     const file = getLibraryFile(versionId, requestedPath);
     if (!file || file.state !== 'ready') return json({ error: 'File is not in a published manifest.' }, { status: 404 });
     const path = await assertRegularPath(file.storageLocation, file.path);

src/lib/server/game-files/index.ts

@@ -1,16 +1,17 @@
 import { randomUUID } from 'node:crypto';
-import { mkdir, rename, stat } from 'node:fs/promises';
+import { mkdir, rename, stat, rm, lstat } from 'node:fs/promises';
 import { dirname, join, resolve } from 'node:path';
 import { db } from '../db/index.js';
 import { config } from '../config.js';
 import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hosts/index.js';
 import { ensureGameFileSchema } from './schema.js';
+import { ensureHostSchema } from '../hosts/schema.js';
 import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js';
 import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock } from './uploads.js';
 import { GAME_FILE_GAMES } from '../../game-files.js';
 import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState, LibraryTransferProgress } from '../../game-files.js';
 
-function database() { const handle = db(); ensureGameFileSchema(handle); return handle; }
+function database() { const handle = db(); ensureGameFileSchema(handle); ensureHostSchema(handle); return handle; }
 export const gameFilesRoot = () => resolve(process.env.GAME_FILES_ROOT ?? join(dirname(config.dbPath), 'game-files'));
 const stagingRoot = (id: string) => join(gameFilesRoot(), '.imports', id);
 type VersionRow = { id: string; library_id: string; game: GameLibraryGame; version: string; description: string; dlc: string; state: GameLibraryVersion['state']; size_bytes: number; manifest: string; manifest_sha256: string | null; created_at: string; published_at: string | null; storage_location: string; source_host_id: string; source_volume: string; error: string | null; operation_id: string | null };
@@ -22,6 +23,8 @@ function versionFromRow(row: VersionRow & { file_count?: number }, summaryCaches
     const files = summaryCaches ? [] : JSON.parse(row.manifest) as GameFileEntry[];
     return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version,
         description: row.description, state: row.state, sizeBytes: row.size_bytes,
+        filesRemoved: !!database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=? AND removed_at IS NOT NULL').get(row.id),
+        filesRemoving: !!database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=? AND removed_at IS NULL').get(row.id),
         files, fileCount: row.file_count ?? (files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count), manifestSha256: row.manifest_sha256, createdAt: row.created_at,
         publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id || null,
         sourceVolume: row.source_volume || null, error: row.error, operationId: row.operation_id,
@@ -31,6 +34,9 @@ export function getLibraryVersion(id: string): GameLibraryVersion | null {
     const row = database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=?').get(id) as VersionRow | undefined;
     return row ? versionFromRow(row) : null;
 }
+export function libraryFilesRetired(versionId: string): boolean {
+    return !!database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(versionId);
+}
 export function getLibraryFile(versionId: string, path: string) {
     validateGamePath(path);
     return database().prepare(`SELECT f.path,f.size,f.sha256,v.state,v.source_host_id AS sourceHostId,v.storage_location AS storageLocation
@@ -75,7 +81,8 @@ export function getHostLibraryCache(hostId: string, versionId: string): HostLibr
 }
 export function requireReadyLibrary(hostId: string, versionId: string): GameLibraryVersion {
     const version = getLibraryVersion(versionId);
-    if (!version || version.state !== 'ready') throw new Error('Select a published game-file version.');
+    if (!version || version.state !== 'ready' || libraryFilesRetired(versionId)) throw new Error('Select an available published game-file version.');
+    if (database().prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.');
     const cache = getHostLibraryCache(hostId, versionId);
     if (!cache || !['ready', 'outdated'].includes(cache.status) || cache.manifestSha256 !== version.manifestSha256) throw new Error('The selected host must finish downloading and verifying this game-file version.');
     return version;
@@ -179,7 +186,7 @@ export async function publishBrowserUpload(id: string): Promise<GameLibraryVersi
         return getLibraryVersion(id)!;
     });
     publishing.set(id, task);
-    try { return await task; } finally { publishing.delete(id); }
+    try { const result = await task; await cleanupControllerLibraries(); return result; } finally { publishing.delete(id); }
 }
 export async function requestBrowserPublication(id: string) {
     const version = requireBrowserUpload(id);
@@ -267,7 +274,7 @@ export async function publishImport(hostId: string, id: string): Promise<GameLib
         return getLibraryVersion(id)!;
     })();
     publishing.set(id, task);
-    try { return await task; } finally { publishing.delete(id); }
+    try { const result = await task; await cleanupControllerLibraries(); return result; } finally { publishing.delete(id); }
 }
 
 /** Large libraries can take minutes to verify. Keep HTTP requests bounded while the source host polls. */
@@ -364,6 +371,8 @@ export function updateHostLibraryCaches(hostId: string, input: unknown, options:
 export function queueLibraryDownload(hostId: string, versionId: string, options: { automatic?: boolean } = {}) {
     const version = getLibraryVersion(versionId);
     if (!version || version.state !== 'ready') throw new Error('Choose a published library version.');
+    if (libraryFilesRetired(versionId)) throw new Error('These outdated game files have been removed. Upload them again to use them.');
+    if (database().prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.');
     const host = getHost(hostId);
     if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled host.');
     const prior = getHostLibraryCache(hostId, versionId);
@@ -410,3 +419,70 @@ export function libraryOverview() {
         })
     })) })), hosts };
 }
+
+/** Any live reservation (including stopped or provisioning instances) pins its files. */
+function libraryInUse(versionId: string, hostId?: string): boolean {
+    const handle = database();
+    // Do not infer an empty fleet if its schema is unavailable.
+    if (!handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_instances'").get()) return true;
+    if (handle.prepare(`SELECT 1 FROM native_instances WHERE library_id=? AND retired_at IS NULL ${hostId ? 'AND host_id=?' : ''} LIMIT 1`)
+        .get(versionId, ...(hostId ? [hostId] : []))) return true;
+    return !!handle.prepare(`SELECT 1 FROM ec_host_operations WHERE state IN ('pending','accepted','running','unknown')
+        AND (json_extract(payload,'$.libraryVersionId')=? OR json_extract(payload,'$.libraryId')=? OR json_extract(payload,'$.importId')=?)
+        ${hostId ? 'AND host_id=?' : ''} LIMIT 1`).get(versionId, versionId, versionId, ...(hostId ? [hostId] : []));
+}
+
+/** Queue serialized, recoverable host cleanup; an offline host keeps its copy pinned. */
+export function reconcileHostLibraryCleanup(hostId: string): void {
+    const handle = database();
+    const host = getHost(hostId);
+    if (!host?.connected || !host.credentialActive || host.mode === 'disabled' || host.decommission
+        || !host.inventory.capabilities?.includes('library.remove')) return;
+    const latest = new Set(latestPublishedLibraries().map(version => version.id));
+    const caches = handle.prepare('SELECT * FROM host_library_caches WHERE host_id=?').all(hostId) as CacheRow[];
+    for (const cache of caches) {
+        if (latest.has(cache.version_id) || (cache.status === 'missing' && cache.bytes_downloaded === 0) || libraryInUse(cache.version_id, hostId)) continue;
+        const previous = handle.prepare(`SELECT state,updated_at FROM ec_host_operations WHERE host_id=? AND kind='library.remove'
+            AND json_extract(payload,'$.libraryVersionId')=? ORDER BY rowid DESC LIMIT 1`).get(hostId, cache.version_id) as { state: string; updated_at: string } | undefined;
+        // Wait for fresh inventory after completion, and back off failures.
+        if (previous && (Date.parse(cache.updated_at) <= Date.parse(previous.updated_at)
+            || previous.state === 'failed' && Date.now() - Date.parse(previous.updated_at) < 60_000)) continue;
+        const version = getLibraryVersion(cache.version_id)!;
+        queueHostOperation(hostId, 'library.remove', { libraryVersionId: version.id, game: version.game });
+    }
+}
+
+const cleaning = new WeakMap<ReturnType<typeof database>, Promise<void>>();
+/** Reserve deletion durably before yielding, so new downloads cannot race filesystem removal. */
+export function cleanupControllerLibraries(): Promise<void> {
+    const handle = database();
+    const current = cleaning.get(handle); if (current) return current;
+    const task = (async () => {
+        const latest = new Set(latestPublishedLibraries().map(version => version.id));
+        const versions = handle.prepare(`SELECT v.id,v.storage_location,l.game FROM game_file_versions v
+            JOIN game_file_libraries l ON l.id=v.library_id LEFT JOIN game_file_cleanup c ON c.version_id=v.id
+            WHERE v.state='ready' AND c.removed_at IS NULL`).all() as { id: string; storage_location: string; game: string }[];
+        for (const version of versions) {
+            if (latest.has(version.id) || libraryInUse(version.id)) continue;
+            // Missing partial downloads still occupy disk. Require a full absence report from every host.
+            if (handle.prepare("SELECT 1 FROM host_library_caches WHERE version_id=? AND (status!='missing' OR bytes_downloaded>0) LIMIT 1").get(version.id)) continue;
+            // Only delete the controller-owned version directory, never an arbitrary stored path.
+            const expected = join(gameFilesRoot(), version.game, version.id);
+            if (!/^[a-zA-Z0-9_-]+$/.test(version.id) || resolve(version.storage_location) !== expected) continue;
+            handle.prepare('INSERT OR IGNORE INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(version.id, new Date().toISOString());
+            try {
+                for (const path of [gameFilesRoot(), dirname(expected), expected]) {
+                    const info = await lstat(path).catch(cause => { if (cause.code === 'ENOENT') return null; throw cause; });
+                    if (info?.isSymbolicLink()) throw new Error('Refusing game-file cleanup through a symlink.');
+                }
+                await rm(expected, { recursive: true, force: true });
+                handle.prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?').run(new Date().toISOString(), version.id);
+            } catch (cause) {
+                console.error(`Game-file cleanup will retry for ${version.id}:`, cause);
+            }
+        }
+    })();
+    cleaning.set(handle, task);
+    void task.finally(() => cleaning.delete(handle)).catch(() => {});
+    return task;
+}

src/lib/server/game-files/schema.ts

@@ -34,6 +34,10 @@ export function ensureGameFileSchema(handle: Database.Database): void {
             size INTEGER NOT NULL, last_modified INTEGER NOT NULL,
             PRIMARY KEY(version_id,path)
         );
+        CREATE TABLE IF NOT EXISTS game_file_cleanup (
+            version_id TEXT PRIMARY KEY REFERENCES game_file_versions(id),
+            started_at TEXT NOT NULL, removed_at TEXT
+        );
         INSERT OR IGNORE INTO game_file_entries(version_id,path,size,sha256)
             SELECT v.id,json_extract(f.value,'$.path'),json_extract(f.value,'$.size'),json_extract(f.value,'$.sha256')
             FROM game_file_versions v,json_each(v.manifest) f;

src/lib/server/pipeline/worker.ts

@@ -16,6 +16,7 @@ import { STATUS_REFRESH_MS } from '../../server-freshness.js';
 import { recordMetrics } from '../fleet/metrics.js';
 import { selectedZombieMap } from '../../map-aliases.js';
 import { runMaintenance } from '../maintenance/index.js';
+import { cleanupControllerLibraries } from '../game-files/index.js';
 import { trackActivity } from './activity.js';
 
 const TICK_MS = 15_000;
@@ -50,13 +51,14 @@ function reconcileRequesters(): void {
     const requesters: [string, string, () => Promise<unknown>][] = [
         ['campaign', 'Reconciling campaigns', sweep],
         ['beta', 'Reconciling beta tests', betaTick],
-        ['maintenance', 'Checking maintenance and recovery', runMaintenance]
+        ['maintenance', 'Checking maintenance and recovery', runMaintenance],
+        ['game-files', 'Removing unused game files', cleanupControllerLibraries]
     ];
     for (const [key, label, work] of requesters) {
         if (requesterFlights.has(key)) continue;
         const flight = trackActivity(key, label, work)
             .catch(err => { console.error(`[worker] ${key} error:`, err); })
-            .then(async () => { await tick(true); })
+            .then(async () => { if (key !== 'game-files') await tick(true); })
             .finally(() => { requesterFlights.delete(key); });
         requesterFlights.set(key, flight);
     }

src/routes/api/agent/heartbeat/+server.ts

@@ -2,6 +2,7 @@ import { json } from '@sveltejs/kit';
 import { heartbeat, pendingCreationCancellations } from '$lib/server/hosts/index.js';
 import { authenticateAgent, hostApiError, objectBody } from '$lib/server/hosts/http.js';
 import { reconcileHostPreparation } from '$lib/server/hosts/preparation.js';
+import { reconcileHostLibraryCleanup, cleanupControllerLibraries } from '$lib/server/game-files/index.js';
 export async function POST({ request }: { request: Request }) {
     const authenticated = authenticateAgent(request);
     try {
@@ -13,6 +14,8 @@ export async function POST({ request }: { request: Request }) {
             updateHostLibraryCaches(host.id, inventory.libraryCaches, { snapshot: true });
         }
         reconcileHostPreparation(host.id);
+        reconcileHostLibraryCleanup(host.id);
+        await cleanupControllerLibraries();
         return json({ host, cancelledCreations: pendingCreationCancellations(host.id), heartbeatSeconds: 15, serverTime: new Date().toISOString() });
     } catch (cause) { return hostApiError(cause); }
 }

src/routes/api/agent/libraries/[id]/manifest/+server.ts

@@ -7,6 +7,7 @@ export function GET({ request, params }: { request: Request; params: { id: strin
         authenticateAgent(request);
         const version = getLibraryVersion(params.id);
         if (!version || version.state !== 'ready') return json({ error: 'Published library not found.' }, { status: 404 });
+        if (version.filesRemoved || version.filesRemoving) return json({ error: 'These outdated game files have been removed.' }, { status: 410 });
         return json({ id: version.id, game: version.game, files: version.files, sizeBytes: version.sizeBytes, manifestSha256: version.manifestSha256 });
     } catch (cause) { return gameFileError(cause); }
 }

src/routes/game-files/+page.svelte

@@ -276,10 +276,11 @@
             {#if history.length}
                 <details class="history">
                     <summary>Previous uploads ({history.length})</summary>
-                    <p class="muted">Previous files remain available for servers that already use them.</p>
+                    <p class="muted">Old host copies are removed once no instances use them. Controller files are removed after all host copies are gone; upload history is kept.</p>
                     {#each history as item (item.id)}
                         <article class="upload-entry">
                             <h3>Uploaded {uploadDate(item)}</h3>
+                            {#if item.filesRemoved}<p class="muted">Files removed from controller · upload history retained</p>{:else if item.filesRemoving}<p class="muted">Removing unused controller files · upload history retained</p>{/if}
                             <p>{formatGameBytes(item.sizeBytes)} · {item.fileCount.toLocaleString()} files</p>
                             {@render uploadDetails(item)}
                             <details><summary>Host copies</summary>{@render hostCopies(item, false)}</details>
@@ -309,7 +310,7 @@
             </tr>
         {:else}<tr><td colspan="4">No host copies yet.</td></tr>{/each}
     </tbody></table></div>
-    {#if canManage}<details><summary>Advanced: download or verify these files</summary><p class="muted">Retry a download or prepare a previous upload on a host. The latest files are downloaded automatically.</p><div class="download"><label>Host<select bind:value={targets[item.id]}><option value="">Select host</option>{#each onlineHosts as host}<option value={host.id}>{host.name} · {formatGameBytes(Number(host.inventory.storageAvailable ?? 0))} free</option>{/each}</select></label><Button size="sm" disabled={busy || !targets[item.id]} onclick={() => download(item.id)}>Download / verify</Button></div></details>{/if}
+    {#if canManage && !item.filesRemoved && !item.filesRemoving}<details><summary>Advanced: download or verify these files</summary><p class="muted">Retry a download or prepare a previous upload on a host. The latest files are downloaded automatically.</p><div class="download"><label>Host<select bind:value={targets[item.id]}><option value="">Select host</option>{#each onlineHosts as host}<option value={host.id}>{host.name} · {formatGameBytes(Number(host.inventory.storageAvailable ?? 0))} free</option>{/each}</select></label><Button size="sm" disabled={busy || !targets[item.id]} onclick={() => download(item.id)}>Download / verify</Button></div></details>{/if}
 {/snippet}
 
 <style>

static/agent/ec-agent.py

@@ -32,7 +32,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.6"
+VERSION = "1.4.7"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 LIBRARY_DOWNLOAD_WORKERS = 4
@@ -1094,7 +1094,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["instance.cancel-create", "operations.read-lane", "operations.health-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "instance.cancel-create", "operations.read-lane", "operations.health-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1722,6 +1722,35 @@ class Agent:
             self.publish_library_progress(payload, timing)
             raise
 
+    def library_remove(self, payload):
+        library_id = identifier(payload.get("libraryVersionId"))
+        directory = self.library_dir(payload.get("game"), library_id)
+        staging = directory.with_name("." + library_id + ".partial")
+        # Never follow substituted storage parents outside the managed tree.
+        for path in (self.root / "libraries", directory.parent, directory, staging):
+            if path.is_symlink():
+                raise Rejected("Refusing library cleanup through a symlink")
+        # Include stopped and unmanaged containers: any bind mount pins the files.
+        # Docker failures must stop cleanup, never masquerade as an empty host.
+        for container_id in self.docker.call("container", "ls", "--all", "--quiet").splitlines():
+            container = self.docker.inspect(container_id)
+            if container is None:
+                raise Rejected("Container inventory changed during library cleanup")
+            for mount in container["Mounts"]:
+                if not mount.get("Source"):  # tmpfs has no host filesystem source.
+                    continue
+                source = Path(mount["Source"]).resolve()
+                for target in (directory.resolve(), staging.resolve()):
+                    if source == target or target in source.parents or source in target.parents:
+                        raise Rejected("Game files are still mounted by a container")
+        self.checkpoint({"phase": "removing-library"})
+        for path in (directory, staging):
+            if path.exists():
+                shutil.rmtree(path)
+        self.library_progress.pop(library_id, None)
+        self.library_transfers.pop(library_id, None)
+        return {"libraryVersionId": library_id, "removed": True}
+
     def library_import(self, payload):
         game = payload.get("game")
         volume = payload.get("sourceVolume")
@@ -1803,6 +1832,8 @@ class Agent:
             return self.inventory()
         if kind == "library.download":
             return self.library_download(payload)
+        if kind == "library.remove":
+            return self.library_remove(payload)
         if kind == "library.import":
             return self.library_import(payload)
         if kind == "library.inspect":
@@ -1923,6 +1954,8 @@ class Agent:
                     return self.journal.update(operation["id"], "unknown", error="Import reconciliation is waiting for controller contact")
                 finally:
                     self.executing_id = None
+            if kind == "library.remove":
+                return self.journal.update(operation["id"], "completed", self.library_remove(payload))
             if kind == "library.download":
                 # This is a resumable immutable transfer, not a repeated
                 # container command. Full files skip transfer; verification

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.4.6",
+  "version": "1.4.7",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "bdedda91475bd29c2140bf32932a1dfc358e7069736439b036ffb333b4300cc3",
+  "sha256": "e9e1b3869d66eca351dadb045d99451b97c91d15c5b7199a3de3241b550de998",
   "updaterProtocolVersion": 1,
   "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }

tests/unit/game-file-library.test.ts

@@ -6,8 +6,8 @@ import { join } from 'node:path';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 const state = vi.hoisted(() => ({ handle: null as Database.Database | null }));
 vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle }));
-import { createEnrollment, createHost, enrollHost, getHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js';
-import { createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, latestPublishedLibraries, libraryOverview, listLibraries, newestReadyLibrary, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js';
+import { heartbeat, createEnrollment, createHost, enrollHost, getHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js';
+import { cleanupControllerLibraries, reconcileHostLibraryCleanup, createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, latestPublishedLibraries, libraryOverview, listLibraries, newestReadyLibrary, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js';
 import { serveGameFile } from '../../src/lib/server/game-files/http.js';
 import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js';
 
@@ -179,3 +179,97 @@ describe('verified host distribution', () => {
         expect(manifestRoute({ request: authorized, params: { id: version.id } }).status).toBe(401);
     });
 });
+
+
+describe('obsolete game-file cleanup', () => {
+    beforeEach(() => {
+        state.handle!.exec('CREATE TABLE native_instances (id TEXT, host_id TEXT, library_id TEXT, retired_at TEXT)');
+    });
+    async function obsolete() {
+        const old = await published();
+        reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
+        const latest = await published();
+        return { old, latest };
+    }
+    it('retains history and the latest files while removing unreferenced old bytes', async () => {
+        const { old, latest } = await obsolete();
+        await cleanupControllerLibraries();
+        expect(getLibraryVersion(old.id)).toMatchObject({ state: 'ready', filesRemoved: true, files: [file] });
+        await expect(readFile(join(old.storageLocation, file.path))).rejects.toThrow();
+        expect(await readFile(join(latest.storageLocation, file.path), 'utf8')).toBe(data);
+        expect(() => queueLibraryDownload(hostId, old.id)).toThrow('removed');
+        expect((await serveGameFile(new Request('http://local/file'), old.id, file.path)).status).toBe(410);
+        await cleanupControllerLibraries();
+        expect(listLibraries()[1].versions).toHaveLength(2);
+    });
+    it('resumes a reserved controller deletion after restart and preserves the manifest', async () => {
+        const old = await published();
+        reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
+        updateHostLibraryCaches(hostId, [{ id: old.id, status: 'ready', manifestSha256: old.manifestSha256 }]);
+        await published();
+        updateHostLibraryCaches(hostId, [], { snapshot: true });
+        state.handle!.prepare('INSERT INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(old.id, new Date().toISOString());
+        await rm(old.storageLocation, { recursive: true });
+        expect(getLibraryVersion(old.id)).toMatchObject({ filesRemoving: true, filesRemoved: false });
+        expect(() => queueLibraryDownload(hostId, old.id)).toThrow('removed');
+        await cleanupControllerLibraries();
+        expect(getLibraryVersion(old.id)).toMatchObject({ filesRemoving: false, filesRemoved: true, files: [file] });
+    });
+    it('does not send cleanup to agents without the capability or erase their cached files', async () => {
+        const old = await published();
+        reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
+        updateHostLibraryCaches(hostId, [{ id: old.id, status: 'ready', manifestSha256: old.manifestSha256 }]);
+        await published();
+        reconcileHostLibraryCleanup(hostId);
+        expect(state.handle!.prepare("SELECT * FROM ec_host_operations WHERE kind='library.remove'").all()).toHaveLength(0);
+        expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
+    });
+    it('pins files for stopped or provisioning instances until retirement', async () => {
+        const old = await published();
+        reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
+        state.handle!.prepare('INSERT INTO native_instances VALUES(?,?,?,NULL)').run('instance', hostId, old.id);
+        await published();
+        await cleanupControllerLibraries();
+        expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
+        state.handle!.exec("UPDATE native_instances SET retired_at='now'");
+        await cleanupControllerLibraries();
+        expect(getLibraryVersion(old.id)?.filesRemoved).toBe(true);
+    });
+    it('waits for offline hosts, partial copies, and unresolved downloads', async () => {
+        const old = await published();
+        reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
+        updateHostLibraryCaches(hostId, [{ id: old.id, status: 'missing', bytesDownloaded: 1 }]);
+        await published();
+        await cleanupControllerLibraries();
+        expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
+        const operation = queueLibraryDownload(hostId, old.id);
+        updateHostLibraryCaches(hostId, [], { snapshot: true });
+        await cleanupControllerLibraries();
+        expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
+        reportHostOperation(hostId, operation.id, { state: 'failed', error: 'interrupted' });
+        await cleanupControllerLibraries();
+        expect(getLibraryVersion(old.id)?.filesRemoved).toBe(true);
+    });
+    it('queues host removal only without instance references and waits for absence before controller cleanup', async () => {
+        heartbeat(hostId, { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, capabilities: ['library.remove'], storageAvailable: 1_000_000_000_000 });
+        const old = await published();
+        reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
+        updateHostLibraryCaches(hostId, [{ id: old.id, status: 'ready', manifestSha256: old.manifestSha256 }]);
+        state.handle!.prepare('INSERT INTO native_instances VALUES(?,?,?,NULL)').run('instance', hostId, old.id);
+        await published();
+        reconcileHostLibraryCleanup(hostId);
+        expect(state.handle!.prepare("SELECT * FROM ec_host_operations WHERE kind='library.remove'").all()).toHaveLength(0);
+        state.handle!.exec("UPDATE native_instances SET retired_at='now'");
+        reconcileHostLibraryCleanup(hostId);
+        reconcileHostLibraryCleanup(hostId);
+        const removals = state.handle!.prepare("SELECT id FROM ec_host_operations WHERE kind='library.remove'").all() as { id: string }[];
+        expect(removals).toHaveLength(1);
+        expect(() => queueLibraryDownload(hostId, old.id)).toThrow('being removed');
+        reportHostOperation(hostId, removals[0].id, { state: 'completed', result: { removed: true } });
+        await cleanupControllerLibraries();
+        expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
+        updateHostLibraryCaches(hostId, [], { snapshot: true });
+        await cleanupControllerLibraries();
+        expect(getLibraryVersion(old.id)?.filesRemoved).toBe(true);
+    });
+});

tests/unit/worker-scheduling.test.ts

@@ -4,6 +4,7 @@ import { newFixture, type Fixture } from '../helpers/fixture.js';
 vi.mock('../../src/lib/server/pipeline/sweep.js', () => ({ sweep: vi.fn() }));
 vi.mock('../../src/lib/server/pipeline/beta-runner.js', () => ({ betaTick: vi.fn() }));
 vi.mock('../../src/lib/server/maintenance/index.js', () => ({ runMaintenance: vi.fn() }));
+vi.mock('../../src/lib/server/game-files/index.js', () => ({ cleanupControllerLibraries: vi.fn(async () => {}) }));
 vi.mock('../../src/lib/server/provisioning/worker.js', () => ({ processProvisioning: vi.fn(async () => {}) }));
 
 function deferred() {