Automate host preparation and speed up game file downloads
This commit is contained in:
@@ -0,0 +1,123 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { config } from '../config.js';
|
||||
import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation } from './index.js';
|
||||
import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js';
|
||||
import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js';
|
||||
|
||||
const activeStates = ['pending', 'accepted', 'running', 'unknown'];
|
||||
type PreparationOperation = { id: string; state: HostOperationState; updated_at: string; error: string | null };
|
||||
type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation };
|
||||
|
||||
function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined {
|
||||
return db().prepare(`SELECT id,state,updated_at,error FROM ec_host_operations
|
||||
WHERE host_id=? AND kind=? AND json_extract(payload,?)=? ORDER BY created_at DESC,rowid DESC LIMIT 1`)
|
||||
.get(hostId, kind, `$.${field}`, value) as PreparationOperation | undefined;
|
||||
}
|
||||
|
||||
function retryAt(hostId: string, target: Target): number {
|
||||
const operation = target.operation;
|
||||
if (!operation || operation.state !== 'failed') return 0;
|
||||
const kind = target.image ? 'runtime.pull' : 'library.download';
|
||||
const field = target.image ? '$.image' : '$.libraryVersionId';
|
||||
const value = target.image ?? target.versionId!;
|
||||
// Count consecutive failures, including across controller restarts. Unknown
|
||||
// operations are never retried under a new ID: their journal must reconcile.
|
||||
const { count } = db().prepare(`SELECT COUNT(*) AS count FROM ec_host_operations
|
||||
WHERE host_id=? AND kind=? AND json_extract(payload,?)=? AND state='failed'
|
||||
AND rowid>COALESCE((SELECT MAX(rowid) FROM ec_host_operations
|
||||
WHERE host_id=? AND kind=? AND json_extract(payload,?)=? AND state='completed'),0)`)
|
||||
.get(hostId, kind, field, value, hostId, kind, field, value) as { count: number };
|
||||
return Date.parse(operation.updated_at) + Math.min(15 * 60_000, 60_000 * 2 ** Math.min(4, Math.max(0, count - 1)));
|
||||
}
|
||||
|
||||
function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string } | null {
|
||||
if (host.decommission) return { state: 'waiting', detail: 'Automatic preparation is paused during host deletion.' };
|
||||
if (host.mode !== 'active') return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
|
||||
if (!host.enrolled) return { state: 'waiting', detail: 'Automatic preparation starts after agent installation.' };
|
||||
if (!host.connected || !host.credentialActive) return { state: 'waiting', detail: 'Waiting for the host to reconnect.' };
|
||||
if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) return { state: 'blocked', detail: 'A compatible host agent is required.' };
|
||||
if (!host.inventory.dockerRunning) return { state: 'blocked', detail: 'Waiting for Docker to become available on the host.' };
|
||||
if (!host.inventory.directoriesWritable) return { state: 'blocked', detail: 'The host cannot write its managed directories.' };
|
||||
return null;
|
||||
}
|
||||
|
||||
function targets(host: Host): Target[] {
|
||||
const image = config.komodo.defaultImage;
|
||||
const runtimeReady = host.inventory.runtimeImages?.includes(image);
|
||||
const result: Target[] = [{ image, requiredBytes: host.minimumFreeBytes,
|
||||
operation: latestOperation(host.id, 'runtime.pull', 'image', image),
|
||||
item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } }];
|
||||
for (const version of latestPublishedLibraries()) {
|
||||
const cache = getHostLibraryCache(host.id, version.id);
|
||||
const matches = cache?.manifestSha256 === version.manifestSha256;
|
||||
const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes;
|
||||
result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
|
||||
operation: latestOperation(host.id, 'library.download', 'libraryVersionId', version.id),
|
||||
item: { kind: 'library', label: `${version.name} · ${version.version}`, state: ready ? 'ready' : 'waiting',
|
||||
detail: ready ? 'Game files are downloaded and verified.' : 'Game files will be downloaded and verified automatically.',
|
||||
bytesDownloaded: cache?.bytesDownloaded ?? 0, bytesTotal: version.sizeBytes } });
|
||||
}
|
||||
const blocker = hostBlocker(host);
|
||||
for (const target of result) {
|
||||
if (target.item.state === 'ready') continue;
|
||||
const operation = target.operation;
|
||||
if (operation) target.item.operationId = operation.id;
|
||||
if (blocker) { Object.assign(target.item, blocker); continue; }
|
||||
if (operation && activeStates.includes(operation.state)) {
|
||||
target.item.state = 'preparing';
|
||||
target.item.detail = operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.'
|
||||
: target.image ? 'Downloading the runtime automatically.' : 'Downloading and verifying game files automatically.';
|
||||
} else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) {
|
||||
target.item.state = 'preparing'; target.item.detail = 'Waiting for a fresh host inventory after preparation.';
|
||||
} else if (retryAt(host.id, target) > Date.now()) {
|
||||
target.item.state = 'blocked';
|
||||
target.item.detail = `${operation?.error ?? 'Preparation failed.'} Automatic retry after ${new Date(retryAt(host.id, target)).toISOString()}.`;
|
||||
} else if (Number(host.inventory.storageAvailable ?? 0) < target.requiredBytes) {
|
||||
target.item.state = 'blocked'; target.item.detail = 'Insufficient free storage, including the host reserve. Preparation resumes automatically when space is available.';
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function hostPreparationStatus(host: Host): HostPreparation {
|
||||
const items = targets(host).map(target => target.item);
|
||||
if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
|
||||
detail: 'Upload and publish a game-file library once; hosts download it automatically.' });
|
||||
const state = items.some(item => item.state === 'blocked') ? 'blocked'
|
||||
: items.some(item => item.state === 'preparing') ? 'preparing'
|
||||
: items.some(item => item.state === 'waiting') ? 'waiting' : 'ready';
|
||||
return { state, items };
|
||||
}
|
||||
|
||||
export function presentHostPreparation(host: Host): Host {
|
||||
const preparation = hostPreparationStatus(host);
|
||||
if (['needs-runtime', 'needs-game-files'].includes(host.readiness) && preparation.state === 'preparing') {
|
||||
return { ...host, readiness: 'preparing', readinessReasons: preparation.items.filter(item => item.state !== 'ready').map(item => item.detail), preparation };
|
||||
}
|
||||
return { ...host, preparation };
|
||||
}
|
||||
|
||||
/** Agent heartbeats and polls drive durable preparation; no open UI or in-memory worker is needed. */
|
||||
export function reconcileHostPreparation(hostId: string): HostPreparation {
|
||||
return db().transaction(() => {
|
||||
const host = getHost(hostId);
|
||||
if (!host) throw new Error('Host was not found.');
|
||||
if (!hostBlocker(host)) {
|
||||
// Queue only one setup mutation at a time so each capacity check uses
|
||||
// fresh disk inventory. This also respects manually queued downloads.
|
||||
const active = db().prepare(`SELECT 1 FROM ec_host_operations WHERE host_id=?
|
||||
AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId);
|
||||
const finished = db().prepare(`SELECT updated_at FROM ec_host_operations WHERE host_id=?
|
||||
AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('completed','failed')
|
||||
ORDER BY updated_at DESC LIMIT 1`).get(hostId) as { updated_at: string } | undefined;
|
||||
if (!active && (!finished || Date.parse(host.lastHeartbeat!) > Date.parse(finished.updated_at))) {
|
||||
// A failed or oversized library must not prevent other libraries
|
||||
// from preparing. The runtime still comes first when eligible.
|
||||
const target = targets(host).find(target => target.item.state === 'waiting');
|
||||
if (target?.image) queueHostOperation(host.id, 'runtime.pull', { image: target.image });
|
||||
else if (target?.versionId) queueLibraryDownload(host.id, target.versionId);
|
||||
}
|
||||
}
|
||||
return hostPreparationStatus(host);
|
||||
}).immediate();
|
||||
}
|
||||
Reference in New Issue
Block a user