josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Automatically remove unused game-file versions

Browse Source
This commit is contained in:
josh
2026-10-04 23:05:32 -04:00
parent 9c6d7cc347
commit 725d41c112
15 changed files with 336 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/game-files/http.ts
+2 -1
View File
@@ -2,7 +2,7 @@ import { createReadStream } from 'node:fs';
import { stat } from 'node:fs/promises';
import { Readable } from 'node:stream';
import { json } from '@sveltejs/kit';
import { getLibraryFile } from './index.js';
import { getLibraryFile, libraryFilesRetired } from './index.js';
import { assertRegularPath, parseByteRange } from './storage.js';
export function gameFileError(cause: unknown): Response {
@@ -28,6 +28,7 @@ export async function readGameJson(request: Request, maxBytes = 64 * 1024): Prom
}
export async function serveGameFile(request: Request, versionId: string, requestedPath: string): Promise<Response> {
if (libraryFilesRetired(versionId)) return json({ error: 'These outdated game files have been removed.' }, { status: 410 });
const file = getLibraryFile(versionId, requestedPath);
if (!file || file.state !== 'ready') return json({ error: 'File is not in a published manifest.' }, { status: 404 });
const path = await assertRegularPath(file.storageLocation, file.path);
src/lib/server/game-files/index.ts
+81 -5
View File
@@ -1,16 +1,17 @@
import { randomUUID } from 'node:crypto';
import { mkdir, rename, stat } from 'node:fs/promises';
import { mkdir, rename, stat, rm, lstat } from 'node:fs/promises';
import { dirname, join, resolve } from 'node:path';
import { db } from '../db/index.js';
import { config } from '../config.js';
import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hosts/index.js';
import { ensureGameFileSchema } from './schema.js';
import { ensureHostSchema } from '../hosts/schema.js';
import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js';
import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock } from './uploads.js';
import { GAME_FILE_GAMES } from '../../game-files.js';
import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState, LibraryTransferProgress } from '../../game-files.js';
function database() { const handle = db(); ensureGameFileSchema(handle); return handle; }
function database() { const handle = db(); ensureGameFileSchema(handle); ensureHostSchema(handle); return handle; }
export const gameFilesRoot = () => resolve(process.env.GAME_FILES_ROOT ?? join(dirname(config.dbPath), 'game-files'));
const stagingRoot = (id: string) => join(gameFilesRoot(), '.imports', id);
type VersionRow = { id: string; library_id: string; game: GameLibraryGame; version: string; description: string; dlc: string; state: GameLibraryVersion['state']; size_bytes: number; manifest: string; manifest_sha256: string | null; created_at: string; published_at: string | null; storage_location: string; source_host_id: string; source_volume: string; error: string | null; operation_id: string | null };
@@ -22,6 +23,8 @@ function versionFromRow(row: VersionRow & { file_count?: number }, summaryCaches
const files = summaryCaches ? [] : JSON.parse(row.manifest) as GameFileEntry[];
return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version,
description: row.description, state: row.state, sizeBytes: row.size_bytes,
filesRemoved: !!database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=? AND removed_at IS NOT NULL').get(row.id),
filesRemoving: !!database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=? AND removed_at IS NULL').get(row.id),
files, fileCount: row.file_count ?? (files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count), manifestSha256: row.manifest_sha256, createdAt: row.created_at,
publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id || null,
sourceVolume: row.source_volume || null, error: row.error, operationId: row.operation_id,
@@ -31,6 +34,9 @@ export function getLibraryVersion(id: string): GameLibraryVersion | null {
const row = database().prepare('SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=?').get(id) as VersionRow | undefined;
return row ? versionFromRow(row) : null;
}
export function libraryFilesRetired(versionId: string): boolean {
return !!database().prepare('SELECT 1 FROM game_file_cleanup WHERE version_id=?').get(versionId);
}
export function getLibraryFile(versionId: string, path: string) {
validateGamePath(path);
return database().prepare(`SELECT f.path,f.size,f.sha256,v.state,v.source_host_id AS sourceHostId,v.storage_location AS storageLocation
@@ -75,7 +81,8 @@ export function getHostLibraryCache(hostId: string, versionId: string): HostLibr
}
export function requireReadyLibrary(hostId: string, versionId: string): GameLibraryVersion {
const version = getLibraryVersion(versionId);
if (!version || version.state !== 'ready') throw new Error('Select a published game-file version.');
if (!version || version.state !== 'ready' || libraryFilesRetired(versionId)) throw new Error('Select an available published game-file version.');
if (database().prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.');
const cache = getHostLibraryCache(hostId, versionId);
if (!cache || !['ready', 'outdated'].includes(cache.status) || cache.manifestSha256 !== version.manifestSha256) throw new Error('The selected host must finish downloading and verifying this game-file version.');
return version;
@@ -179,7 +186,7 @@ export async function publishBrowserUpload(id: string): Promise<GameLibraryVersi
return getLibraryVersion(id)!;
});
publishing.set(id, task);
try { return await task; } finally { publishing.delete(id); }
try { const result = await task; await cleanupControllerLibraries(); return result; } finally { publishing.delete(id); }
}
export async function requestBrowserPublication(id: string) {
const version = requireBrowserUpload(id);
@@ -267,7 +274,7 @@ export async function publishImport(hostId: string, id: string): Promise<GameLib
return getLibraryVersion(id)!;
})();
publishing.set(id, task);
try { return await task; } finally { publishing.delete(id); }
try { const result = await task; await cleanupControllerLibraries(); return result; } finally { publishing.delete(id); }
}
/** Large libraries can take minutes to verify. Keep HTTP requests bounded while the source host polls. */
@@ -364,6 +371,8 @@ export function updateHostLibraryCaches(hostId: string, input: unknown, options:
export function queueLibraryDownload(hostId: string, versionId: string, options: { automatic?: boolean } = {}) {
const version = getLibraryVersion(versionId);
if (!version || version.state !== 'ready') throw new Error('Choose a published library version.');
if (libraryFilesRetired(versionId)) throw new Error('These outdated game files have been removed. Upload them again to use them.');
if (database().prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.');
const host = getHost(hostId);
if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled host.');
const prior = getHostLibraryCache(hostId, versionId);
@@ -410,3 +419,70 @@ export function libraryOverview() {
})
})) })), hosts };
}
/** Any live reservation (including stopped or provisioning instances) pins its files. */
function libraryInUse(versionId: string, hostId?: string): boolean {
const handle = database();
// Do not infer an empty fleet if its schema is unavailable.
if (!handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_instances'").get()) return true;
if (handle.prepare(`SELECT 1 FROM native_instances WHERE library_id=? AND retired_at IS NULL ${hostId ? 'AND host_id=?' : ''} LIMIT 1`)
.get(versionId, ...(hostId ? [hostId] : []))) return true;
return !!handle.prepare(`SELECT 1 FROM ec_host_operations WHERE state IN ('pending','accepted','running','unknown')
AND (json_extract(payload,'$.libraryVersionId')=? OR json_extract(payload,'$.libraryId')=? OR json_extract(payload,'$.importId')=?)
${hostId ? 'AND host_id=?' : ''} LIMIT 1`).get(versionId, versionId, versionId, ...(hostId ? [hostId] : []));
}
/** Queue serialized, recoverable host cleanup; an offline host keeps its copy pinned. */
export function reconcileHostLibraryCleanup(hostId: string): void {
const handle = database();
const host = getHost(hostId);
if (!host?.connected || !host.credentialActive || host.mode === 'disabled' || host.decommission
|| !host.inventory.capabilities?.includes('library.remove')) return;
const latest = new Set(latestPublishedLibraries().map(version => version.id));
const caches = handle.prepare('SELECT * FROM host_library_caches WHERE host_id=?').all(hostId) as CacheRow[];
for (const cache of caches) {
if (latest.has(cache.version_id) || (cache.status === 'missing' && cache.bytes_downloaded === 0) || libraryInUse(cache.version_id, hostId)) continue;
const previous = handle.prepare(`SELECT state,updated_at FROM ec_host_operations WHERE host_id=? AND kind='library.remove'
AND json_extract(payload,'$.libraryVersionId')=? ORDER BY rowid DESC LIMIT 1`).get(hostId, cache.version_id) as { state: string; updated_at: string } | undefined;
// Wait for fresh inventory after completion, and back off failures.
if (previous && (Date.parse(cache.updated_at) <= Date.parse(previous.updated_at)
|| previous.state === 'failed' && Date.now() - Date.parse(previous.updated_at) < 60_000)) continue;
const version = getLibraryVersion(cache.version_id)!;
queueHostOperation(hostId, 'library.remove', { libraryVersionId: version.id, game: version.game });
}
}
const cleaning = new WeakMap<ReturnType<typeof database>, Promise<void>>();
/** Reserve deletion durably before yielding, so new downloads cannot race filesystem removal. */
export function cleanupControllerLibraries(): Promise<void> {
const handle = database();
const current = cleaning.get(handle); if (current) return current;
const task = (async () => {
const latest = new Set(latestPublishedLibraries().map(version => version.id));
const versions = handle.prepare(`SELECT v.id,v.storage_location,l.game FROM game_file_versions v
JOIN game_file_libraries l ON l.id=v.library_id LEFT JOIN game_file_cleanup c ON c.version_id=v.id
WHERE v.state='ready' AND c.removed_at IS NULL`).all() as { id: string; storage_location: string; game: string }[];
for (const version of versions) {
if (latest.has(version.id) || libraryInUse(version.id)) continue;
// Missing partial downloads still occupy disk. Require a full absence report from every host.
if (handle.prepare("SELECT 1 FROM host_library_caches WHERE version_id=? AND (status!='missing' OR bytes_downloaded>0) LIMIT 1").get(version.id)) continue;
// Only delete the controller-owned version directory, never an arbitrary stored path.
const expected = join(gameFilesRoot(), version.game, version.id);
if (!/^[a-zA-Z0-9_-]+$/.test(version.id) || resolve(version.storage_location) !== expected) continue;
handle.prepare('INSERT OR IGNORE INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(version.id, new Date().toISOString());
try {
for (const path of [gameFilesRoot(), dirname(expected), expected]) {
const info = await lstat(path).catch(cause => { if (cause.code === 'ENOENT') return null; throw cause; });
if (info?.isSymbolicLink()) throw new Error('Refusing game-file cleanup through a symlink.');
}
await rm(expected, { recursive: true, force: true });
handle.prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?').run(new Date().toISOString(), version.id);
} catch (cause) {
console.error(`Game-file cleanup will retry for ${version.id}:`, cause);
}
}
})();
cleaning.set(handle, task);
void task.finally(() => cleaning.delete(handle)).catch(() => {});
return task;
}
src/lib/server/game-files/schema.ts
+4
View File
@@ -34,6 +34,10 @@ export function ensureGameFileSchema(handle: Database.Database): void {
size INTEGER NOT NULL, last_modified INTEGER NOT NULL,
PRIMARY KEY(version_id,path)
);
CREATE TABLE IF NOT EXISTS game_file_cleanup (
version_id TEXT PRIMARY KEY REFERENCES game_file_versions(id),
started_at TEXT NOT NULL, removed_at TEXT
);
INSERT OR IGNORE INTO game_file_entries(version_id,path,size,sha256)
SELECT v.id,json_extract(f.value,'$.path'),json_extract(f.value,'$.size'),json_extract(f.value,'$.sha256')
FROM game_file_versions v,json_each(v.manifest) f;