josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Separate game file downloads and verification with progress timing

Browse Source
This commit is contained in:
josh
2026-10-04 12:51:05 -04:00
parent 68323c0baa
commit f8c60fd101
17 changed files with 1150 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/game-files/index.ts
+45 -9
View File
@@ -8,14 +8,16 @@ import { ensureGameFileSchema } from './schema.js';
import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js';
import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock } from './uploads.js';
import { GAME_FILE_GAMES } from '../../game-files.js';
import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState } from '../../game-files.js';
import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState, LibraryTransferProgress } from '../../game-files.js';
function database() { const handle = db(); ensureGameFileSchema(handle); return handle; }
export const gameFilesRoot = () => resolve(process.env.GAME_FILES_ROOT ?? join(dirname(config.dbPath), 'game-files'));
const stagingRoot = (id: string) => join(gameFilesRoot(), '.imports', id);
type VersionRow = { id: string; library_id: string; game: GameLibraryGame; version: string; description: string; dlc: string; state: GameLibraryVersion['state']; size_bytes: number; manifest: string; manifest_sha256: string | null; created_at: string; published_at: string | null; storage_location: string; source_host_id: string; source_volume: string; error: string | null; operation_id: string | null };
type CacheRow = { host_id: string; version_id: string; status: LibraryCacheState; bytes_downloaded: number; bytes_total: number; manifest_sha256: string | null; error: string | null; updated_at: string; operation_id: string | null };
const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ hostId: row.host_id, libraryVersionId: row.version_id, status: row.status, bytesDownloaded: row.bytes_downloaded, bytesTotal: row.bytes_total, manifestSha256: row.manifest_sha256, error: row.error, updatedAt: row.updated_at, operationId: row.operation_id });
type CacheRow = { host_id: string; version_id: string; status: LibraryCacheState; bytes_downloaded: number; bytes_total: number; manifest_sha256: string | null; error: string | null; updated_at: string; operation_id: string | null; bytes_verified: number | null; transfer_progress: string | null };
const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ hostId: row.host_id, libraryVersionId: row.version_id, status: row.status, bytesDownloaded: row.bytes_downloaded, bytesTotal: row.bytes_total, manifestSha256: row.manifest_sha256, error: row.error, updatedAt: row.updated_at, operationId: row.operation_id,
...(row.bytes_verified == null ? {} : { bytesVerified: row.bytes_verified }),
...(row.transfer_progress ? { transfer: JSON.parse(row.transfer_progress) as LibraryTransferProgress } : {}) });
function versionFromRow(row: VersionRow): GameLibraryVersion {
const files = JSON.parse(row.manifest) as GameFileEntry[];
return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version,
@@ -282,6 +284,28 @@ export function requestImportPublication(hostId: string, id: string) {
return importPublicationStatus(hostId, id);
}
function transferProgress(value: unknown, status: LibraryCacheState): LibraryTransferProgress | null {
if (value === null || value === undefined) return null;
if (typeof value !== 'object' || Array.isArray(value)) throw new Error('Invalid transfer progress.');
const progress = value as Record<string, unknown>;
const nonnegative = (amount: unknown): amount is number => typeof amount === 'number' && Number.isFinite(amount) && amount >= 0 && amount <= Number.MAX_SAFE_INTEGER;
if (!['downloading', 'verifying'].includes(progress.phase as string)
|| typeof progress.startedAt !== 'string' || progress.startedAt.length > 80
|| !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})$/.test(progress.startedAt)
|| !Number.isFinite(Date.parse(progress.startedAt))
|| !nonnegative(progress.elapsedSeconds) || !nonnegative(progress.phaseElapsedSeconds)
|| progress.phaseElapsedSeconds > progress.elapsedSeconds
|| (progress.bytesPerSecond != null && !nonnegative(progress.bytesPerSecond))
|| (progress.etaSeconds != null && !nonnegative(progress.etaSeconds))
|| (['downloading', 'verifying'].includes(status) && progress.phase !== status)
|| (['ready', 'outdated'].includes(status) && progress.phase !== 'verifying')) {
throw new Error('Invalid transfer progress.');
}
return { phase: progress.phase as LibraryTransferProgress['phase'], startedAt: progress.startedAt,
elapsedSeconds: progress.elapsedSeconds, phaseElapsedSeconds: progress.phaseElapsedSeconds,
bytesPerSecond: progress.bytesPerSecond as number | null ?? null, etaSeconds: progress.etaSeconds as number | null ?? null };
}
export function updateHostLibraryCaches(hostId: string, input: unknown, options: { snapshot?: boolean } = {}): void {
if (!Array.isArray(input) || input.length > 2000) throw new Error('Invalid host library report.');
const handle = database();
@@ -303,15 +327,27 @@ export function updateHostLibraryCaches(hostId: string, input: unknown, options:
const bytes = raw.bytesDownloaded ?? (['ready', 'outdated'].includes(status) ? version.sizeBytes : 0);
if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > version.sizeBytes) throw new Error('Invalid library download progress.');
if (['ready', 'outdated'].includes(status) && bytes !== version.sizeBytes) { status = 'failed'; error = 'Host reported an incomplete library as verified.'; }
handle.prepare(`INSERT INTO host_library_caches(host_id,version_id,status,bytes_downloaded,bytes_total,manifest_sha256,error,updated_at)
VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(host_id,version_id) DO UPDATE SET status=excluded.status,
bytes_downloaded=excluded.bytes_downloaded,bytes_total=excluded.bytes_total,manifest_sha256=excluded.manifest_sha256,error=excluded.error,updated_at=excluded.updated_at`)
.run(hostId, id, status, bytes, version.sizeBytes, checksum, error, now);
const prior = getHostLibraryCache(hostId, id);
const terminal = ['ready', 'outdated', 'failed'].includes(status);
// Older agents do not report timing. Keep a completed measurement on
// later inventory heartbeats, but never carry an ETA into a new phase.
let transfer = raw.transfer === undefined && terminal ? prior?.transfer ?? null : transferProgress(raw.transfer, status);
let verified = raw.bytesVerified ?? (['ready', 'outdated'].includes(status) ? version.sizeBytes : terminal ? prior?.bytesVerified ?? null : null);
if (verified !== null && (!Number.isSafeInteger(verified) || verified < 0 || verified > version.sizeBytes)) throw new Error('Invalid library verification progress.');
if (transfer && status === 'verifying' && bytes !== version.sizeBytes) throw new Error('Download must finish before verification starts.');
if (['ready', 'outdated'].includes(status) && verified !== version.sizeBytes) throw new Error('Host reported incomplete verification as ready.');
if (status === 'missing') { transfer = null; verified = null; }
if (terminal && transfer) transfer = { ...transfer, bytesPerSecond: null, etaSeconds: null };
handle.prepare(`INSERT INTO host_library_caches(host_id,version_id,status,bytes_downloaded,bytes_total,manifest_sha256,error,updated_at,bytes_verified,transfer_progress)
VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(host_id,version_id) DO UPDATE SET status=excluded.status,
bytes_downloaded=excluded.bytes_downloaded,bytes_total=excluded.bytes_total,manifest_sha256=excluded.manifest_sha256,error=excluded.error,updated_at=excluded.updated_at,
bytes_verified=excluded.bytes_verified,transfer_progress=excluded.transfer_progress`)
.run(hostId, id, status, bytes, version.sizeBytes, checksum, error, now, verified, transfer ? JSON.stringify(transfer) : null);
}
if (options.snapshot) {
const cached = handle.prepare('SELECT version_id FROM host_library_caches WHERE host_id=?').all(hostId) as { version_id: string }[];
for (const cache of cached) if (!reported.has(cache.version_id)) {
handle.prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=? AND version_id=?")
handle.prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,bytes_verified=NULL,transfer_progress=NULL,updated_at=? WHERE host_id=? AND version_id=?")
.run(now, hostId, cache.version_id);
}
}
@@ -336,7 +372,7 @@ export function queueLibraryDownload(hostId: string, versionId: string, options:
manifestSha256: version.manifestSha256, sizeBytes: version.sizeBytes,
...(options.automatic ? { automatic: true } : {})
});
updateHostLibraryCaches(hostId, [{ libraryVersionId: versionId, status: 'downloading', bytesDownloaded: retainedBytes, manifestSha256: prior?.manifestSha256 }]);
updateHostLibraryCaches(hostId, [{ libraryVersionId: versionId, status: 'downloading', bytesDownloaded: retainedBytes, manifestSha256: prior?.manifestSha256, bytesVerified: 0, transfer: null }]);
database().prepare('UPDATE host_library_caches SET operation_id=? WHERE host_id=? AND version_id=?').run(operation.id, hostId, versionId);
return operation;
}