Automate host preparation and speed up game file downloads

src/lib/server/game-files/index.ts

@@ -38,6 +38,14 @@ export function newestReadyLibrary(game: string): GameLibraryVersion | null {
     const row = database().prepare("SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.game=? AND v.state='ready' ORDER BY v.created_at DESC, v.rowid DESC LIMIT 1").get(game) as VersionRow | undefined;
     return row ? versionFromRow(row) : null;
 }
+/** Preparation needs only the latest published version of each library, never its full manifest. */
+export function latestPublishedLibraries() {
+    return database().prepare(`SELECT v.id, l.name, l.game, v.version, v.size_bytes AS sizeBytes, v.manifest_sha256 AS manifestSha256
+        FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
+        WHERE v.state='ready' AND v.id=(SELECT newest.id FROM game_file_versions newest
+            WHERE newest.library_id=l.id AND newest.state='ready' ORDER BY newest.created_at DESC,newest.rowid DESC LIMIT 1)
+        ORDER BY l.name`).all() as { id: string; name: string; game: GameLibraryGame; version: string; sizeBytes: number; manifestSha256: string }[];
+}
 export function listLibraries(): GameLibrary[] {
     reconcileLibraryOperations();
     return (database().prepare('SELECT * FROM game_file_libraries ORDER BY name').all() as Array<{ id: string; name: string; game: GameLibraryGame; description: string; created_at: string }>).map(row => ({
@@ -315,18 +323,19 @@ export function queueLibraryDownload(hostId: string, versionId: string) {
     if (!version || version.state !== 'ready') throw new Error('Choose a published library version.');
     const host = getHost(hostId);
     if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled host.');
-    if (Number(host.inventory.storageAvailable ?? 0) < version.sizeBytes + host.minimumFreeBytes) throw new Error('This host has insufficient storage for this library and its minimum free-space reserve.');
     const prior = getHostLibraryCache(hostId, versionId);
     if (prior?.operationId) {
         const operation = getHostOperation(prior.operationId);
         if (operation && ['pending', 'accepted', 'running', 'unknown'].includes(operation.state)) return operation;
     }
+    const retainedBytes = prior?.manifestSha256 === version.manifestSha256 ? prior.bytesDownloaded : 0;
+    if (Number(host.inventory.storageAvailable ?? 0) < version.sizeBytes - retainedBytes + host.minimumFreeBytes) throw new Error('This host has insufficient storage for this library and its minimum free-space reserve.');
     const operation = queueHostOperation(hostId, 'library.download', {
         libraryVersionId: versionId, game: version.game, manifestUrl: `/api/agent/libraries/${versionId}/manifest`,
         filesBaseUrl: `/api/agent/libraries/${versionId}/files`, statusUrl: `/api/agent/libraries/${versionId}/status`,
         manifestSha256: version.manifestSha256, sizeBytes: version.sizeBytes
     });
-    updateHostLibraryCaches(hostId, [{ libraryVersionId: versionId, status: 'downloading', bytesDownloaded: 0 }]);
+    updateHostLibraryCaches(hostId, [{ libraryVersionId: versionId, status: 'downloading', bytesDownloaded: retainedBytes, manifestSha256: prior?.manifestSha256 }]);
     database().prepare('UPDATE host_library_caches SET operation_id=? WHERE host_id=? AND version_id=?').run(operation.id, hostId, versionId);
     return operation;
 }

src/lib/server/hosts/index.ts

@@ -33,8 +33,8 @@ function hostFromRow(row: HostRow): Host {
     const infrastructureReady = reasons.length === 0;
     const runtimeReady = !!inventory.runtimeImages?.length;
     const libraryReady = !!inventory.libraries?.some(library => library.state === 'ready');
-    if (!runtimeReady) reasons.push('Prepare a Plutainer runtime image.');
-    if (!libraryReady) reasons.push('Download and verify a game-file library.');
+    if (!runtimeReady) reasons.push('The Plutainer runtime image is not ready yet.');
+    if (!libraryReady) reasons.push('No verified game-file library is available yet.');
     if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
     if (row.mode === 'disabled') reasons.push('Host is disabled.');
     const decommission = presentDecommission(decommissionRow(row.id));

src/lib/server/hosts/preparation.ts

@@ -0,0 +1,123 @@
+import { db } from '../db/index.js';
+import { config } from '../config.js';
+import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation } from './index.js';
+import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js';
+import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js';
+
+const activeStates = ['pending', 'accepted', 'running', 'unknown'];
+type PreparationOperation = { id: string; state: HostOperationState; updated_at: string; error: string | null };
+type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation };
+
+function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined {
+    return db().prepare(`SELECT id,state,updated_at,error FROM ec_host_operations
+        WHERE host_id=? AND kind=? AND json_extract(payload,?)=? ORDER BY created_at DESC,rowid DESC LIMIT 1`)
+        .get(hostId, kind, `$.${field}`, value) as PreparationOperation | undefined;
+}
+
+function retryAt(hostId: string, target: Target): number {
+    const operation = target.operation;
+    if (!operation || operation.state !== 'failed') return 0;
+    const kind = target.image ? 'runtime.pull' : 'library.download';
+    const field = target.image ? '$.image' : '$.libraryVersionId';
+    const value = target.image ?? target.versionId!;
+    // Count consecutive failures, including across controller restarts. Unknown
+    // operations are never retried under a new ID: their journal must reconcile.
+    const { count } = db().prepare(`SELECT COUNT(*) AS count FROM ec_host_operations
+        WHERE host_id=? AND kind=? AND json_extract(payload,?)=? AND state='failed'
+        AND rowid>COALESCE((SELECT MAX(rowid) FROM ec_host_operations
+            WHERE host_id=? AND kind=? AND json_extract(payload,?)=? AND state='completed'),0)`)
+        .get(hostId, kind, field, value, hostId, kind, field, value) as { count: number };
+    return Date.parse(operation.updated_at) + Math.min(15 * 60_000, 60_000 * 2 ** Math.min(4, Math.max(0, count - 1)));
+}
+
+function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string } | null {
+    if (host.decommission) return { state: 'waiting', detail: 'Automatic preparation is paused during host deletion.' };
+    if (host.mode !== 'active') return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
+    if (!host.enrolled) return { state: 'waiting', detail: 'Automatic preparation starts after agent installation.' };
+    if (!host.connected || !host.credentialActive) return { state: 'waiting', detail: 'Waiting for the host to reconnect.' };
+    if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) return { state: 'blocked', detail: 'A compatible host agent is required.' };
+    if (!host.inventory.dockerRunning) return { state: 'blocked', detail: 'Waiting for Docker to become available on the host.' };
+    if (!host.inventory.directoriesWritable) return { state: 'blocked', detail: 'The host cannot write its managed directories.' };
+    return null;
+}
+
+function targets(host: Host): Target[] {
+    const image = config.komodo.defaultImage;
+    const runtimeReady = host.inventory.runtimeImages?.includes(image);
+    const result: Target[] = [{ image, requiredBytes: host.minimumFreeBytes,
+        operation: latestOperation(host.id, 'runtime.pull', 'image', image),
+        item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } }];
+    for (const version of latestPublishedLibraries()) {
+        const cache = getHostLibraryCache(host.id, version.id);
+        const matches = cache?.manifestSha256 === version.manifestSha256;
+        const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes;
+        result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
+            operation: latestOperation(host.id, 'library.download', 'libraryVersionId', version.id),
+            item: { kind: 'library', label: `${version.name} · ${version.version}`, state: ready ? 'ready' : 'waiting',
+                detail: ready ? 'Game files are downloaded and verified.' : 'Game files will be downloaded and verified automatically.',
+                bytesDownloaded: cache?.bytesDownloaded ?? 0, bytesTotal: version.sizeBytes } });
+    }
+    const blocker = hostBlocker(host);
+    for (const target of result) {
+        if (target.item.state === 'ready') continue;
+        const operation = target.operation;
+        if (operation) target.item.operationId = operation.id;
+        if (blocker) { Object.assign(target.item, blocker); continue; }
+        if (operation && activeStates.includes(operation.state)) {
+            target.item.state = 'preparing';
+            target.item.detail = operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.'
+                : target.image ? 'Downloading the runtime automatically.' : 'Downloading and verifying game files automatically.';
+        } else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) {
+            target.item.state = 'preparing'; target.item.detail = 'Waiting for a fresh host inventory after preparation.';
+        } else if (retryAt(host.id, target) > Date.now()) {
+            target.item.state = 'blocked';
+            target.item.detail = `${operation?.error ?? 'Preparation failed.'} Automatic retry after ${new Date(retryAt(host.id, target)).toISOString()}.`;
+        } else if (Number(host.inventory.storageAvailable ?? 0) < target.requiredBytes) {
+            target.item.state = 'blocked'; target.item.detail = 'Insufficient free storage, including the host reserve. Preparation resumes automatically when space is available.';
+        }
+    }
+    return result;
+}
+
+export function hostPreparationStatus(host: Host): HostPreparation {
+    const items = targets(host).map(target => target.item);
+    if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
+        detail: 'Upload and publish a game-file library once; hosts download it automatically.' });
+    const state = items.some(item => item.state === 'blocked') ? 'blocked'
+        : items.some(item => item.state === 'preparing') ? 'preparing'
+        : items.some(item => item.state === 'waiting') ? 'waiting' : 'ready';
+    return { state, items };
+}
+
+export function presentHostPreparation(host: Host): Host {
+    const preparation = hostPreparationStatus(host);
+    if (['needs-runtime', 'needs-game-files'].includes(host.readiness) && preparation.state === 'preparing') {
+        return { ...host, readiness: 'preparing', readinessReasons: preparation.items.filter(item => item.state !== 'ready').map(item => item.detail), preparation };
+    }
+    return { ...host, preparation };
+}
+
+/** Agent heartbeats and polls drive durable preparation; no open UI or in-memory worker is needed. */
+export function reconcileHostPreparation(hostId: string): HostPreparation {
+    return db().transaction(() => {
+        const host = getHost(hostId);
+        if (!host) throw new Error('Host was not found.');
+        if (!hostBlocker(host)) {
+            // Queue only one setup mutation at a time so each capacity check uses
+            // fresh disk inventory. This also respects manually queued downloads.
+            const active = db().prepare(`SELECT 1 FROM ec_host_operations WHERE host_id=?
+                AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId);
+            const finished = db().prepare(`SELECT updated_at FROM ec_host_operations WHERE host_id=?
+                AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('completed','failed')
+                ORDER BY updated_at DESC LIMIT 1`).get(hostId) as { updated_at: string } | undefined;
+            if (!active && (!finished || Date.parse(host.lastHeartbeat!) > Date.parse(finished.updated_at))) {
+                // A failed or oversized library must not prevent other libraries
+                // from preparing. The runtime still comes first when eligible.
+                const target = targets(host).find(target => target.item.state === 'waiting');
+                if (target?.image) queueHostOperation(host.id, 'runtime.pull', { image: target.image });
+                else if (target?.versionId) queueLibraryDownload(host.id, target.versionId);
+            }
+        }
+        return hostPreparationStatus(host);
+    }).immediate();
+}