import { db } from '../db/index.js'; import { config } from '../config.js'; import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation } from './index.js'; import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js'; import { GAME_FILE_GAMES } from '../../game-files.js'; import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js'; import { scheduleAvailabilityRefresh } from './availability.js'; const activeStates = ['pending', 'accepted', 'running', 'unknown']; const supersededDownload = 'Superseded by newer game files before delivery to the host.'; type PreparationOperation = { id: string; sequence: number; state: HostOperationState; updated_at: string; error: string | null }; type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation }; type QueuedSelection = { libraryId: string; image: string | null }; type LibraryTarget = Pick[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean }; /** Accepted requests retain their prerequisites until placement owns an instance. */ function queuedSelections(hostId: string): QueuedSelection[] { // Standalone host/library readers can initialize before provisioning migrations. if (!db().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'").get()) return []; return db().prepare(`SELECT DISTINCT json_extract(q.selection,'$.libraryId') AS libraryId, json_extract(q.selection,'$.image') AS image FROM native_creation_intents q JOIN server_deployments d ON d.id=q.deployment_id LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.action='create' AND d.server_deleted_at IS NULL AND n.deployment_id IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND (json_extract(q.selection,'$.automatic')=1 OR json_extract(q.selection,'$.hostId') IS NULL OR json_extract(q.selection,'$.hostId')=?) UNION SELECT i.library_id AS libraryId,i.image FROM ec_host_migrations m JOIN native_instances i ON i.id=m.instance_id JOIN server_deployments d ON d.id=m.deployment_id WHERE m.removed_at IS NULL AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND (m.target_host_id IS NULL OR m.target_host_id=?)`).all(hostId, hostId) as QueuedSelection[]; } function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] { const latest = latestPublishedLibraries(); const latestIds = new Set(latest.map(version => version.id)); const queued = new Map(); const get = db().prepare(`SELECT v.id,l.game,v.version,v.size_bytes AS sizeBytes,v.manifest_sha256 AS manifestSha256 FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE v.id=? AND v.state='ready' AND NOT EXISTS(SELECT 1 FROM game_file_cleanup c WHERE c.version_id=v.id)`); for (const selection of selections) { if (latestIds.has(selection.libraryId) || queued.has(selection.libraryId)) continue; const version = get.get(selection.libraryId) as { id: string; game: string; version: string; sizeBytes: number; manifestSha256: string } | undefined; if (version) queued.set(version.id, { ...version, queued: true, name: `${GAME_FILE_GAMES.find(game => game.id === version.game)?.name ?? version.game} ยท Queued version ${version.version}` }); } // Finish prerequisites for accepted work before downloading newer optional files. return [...queued.values(), ...latest]; } function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined { return db().prepare(`SELECT id,rowid AS sequence,state,updated_at,error FROM ec_host_operations WHERE host_id=? AND kind=? AND json_extract(payload,?)=? ORDER BY created_at DESC,rowid DESC LIMIT 1`) .get(hostId, kind, `$.${field}`, value) as PreparationOperation | undefined; } function retryAt(hostId: string, target: Target): number { const operation = target.operation; if (!operation || operation.state !== 'failed') return 0; const kind = target.image ? 'runtime.pull' : 'library.download'; const field = target.image ? '$.image' : '$.libraryVersionId'; const value = target.image ?? target.versionId!; // Count consecutive failures, including across controller restarts. Unknown // operations are never retried under a new ID: their journal must reconcile. const { count } = db().prepare(`SELECT COUNT(*) AS count FROM ec_host_operations WHERE host_id=? AND kind=? AND json_extract(payload,?)=? AND state='failed' AND rowid>COALESCE((SELECT MAX(rowid) FROM ec_host_operations WHERE host_id=? AND kind=? AND json_extract(payload,?)=? AND state='completed'),0)`) .get(hostId, kind, field, value, hostId, kind, field, value) as { count: number }; return Date.parse(operation.updated_at) + Math.min(15 * 60_000, 60_000 * 2 ** Math.min(4, Math.max(0, count - 1))); } function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string } | null { if (host.decommission) return { state: 'waiting', detail: 'Automatic preparation is paused during host deletion.' }; if (host.mode !== 'active' && !host.enablingSince) return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` }; if (!host.enrolled) return { state: 'waiting', detail: 'Automatic preparation starts after agent installation.' }; if (!host.connected || !host.credentialActive) return { state: 'waiting', detail: 'Waiting for the host to reconnect.' }; if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) return { state: 'blocked', detail: 'A compatible host agent is required.' }; if (!host.inventory.dockerRunning) return { state: 'blocked', detail: 'Waiting for Docker to become available on the host.' }; if (!host.inventory.directoriesWritable) return { state: 'blocked', detail: 'The host cannot write its managed directories.' }; return null; } function targets(host: Host): Target[] { const enabling = db().prepare('SELECT operation_floor FROM ec_host_enable_requests WHERE host_id=?').get(host.id) as { operation_floor: number } | undefined; const verifiedThisEnable = (operation: PreparationOperation | undefined) => !enabling || !!operation && operation.sequence > enabling.operation_floor && operation.state === 'completed' && Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at); const selections = queuedSelections(host.id); const images = new Set([config.komodo.defaultImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]); const result: Target[] = [...images].map(image => { const operation = latestOperation(host.id, 'runtime.pull', 'image', image); const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation); return { image, requiredBytes: host.minimumFreeBytes, operation, item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } }; }); for (const version of targetLibraries(selections)) { const cache = getHostLibraryCache(host.id, version.id); const matches = cache?.manifestSha256 === version.manifestSha256; const operation = latestOperation(host.id, 'library.download', 'libraryVersionId', version.id); const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes && verifiedThisEnable(operation); result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes, operation, item: { kind: 'library', label: version.name, state: ready ? 'ready' : 'waiting', detail: version.queued ? ready ? 'Game files selected by queued launches are downloaded and verified.' : 'Game files selected by queued launches will be downloaded and verified automatically.' : ready ? 'The latest game files are downloaded and verified.' : 'The latest game files will be downloaded and verified automatically.', bytesDownloaded: cache?.bytesDownloaded ?? 0, bytesTotal: version.sizeBytes, ...(cache ? { cache } : {}) } }); } const blocker = hostBlocker(host); for (const target of result) { if (target.item.state === 'ready') continue; const operation = target.operation; if (operation) target.item.operationId = operation.id; if (blocker) { Object.assign(target.item, blocker); continue; } if (operation && activeStates.includes(operation.state)) { target.item.state = 'preparing'; target.item.detail = operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.' : target.image ? 'Downloading the runtime automatically.' : target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.'; } else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) { target.item.state = 'preparing'; target.item.detail = 'Waiting for a fresh host inventory after preparation.'; } else if ((!enabling || (operation?.sequence ?? 0) > enabling.operation_floor) && retryAt(host.id, target) > Date.now()) { target.item.state = 'blocked'; target.item.detail = `${operation?.error ?? 'Preparation failed.'} Automatic retry after ${new Date(retryAt(host.id, target)).toISOString()}.`; } else if (Number(host.inventory.storageAvailable ?? 0) < target.requiredBytes) { target.item.state = 'blocked'; target.item.detail = 'Insufficient free storage, including the host reserve. Preparation resumes automatically when space is available.'; } } return result; } export function hostPreparationStatus(host: Host): HostPreparation { const items = targets(host).map(target => target.item); if (host.enablingSince) { const published = latestPublishedLibraries(); for (const game of GAME_FILE_GAMES) if (!published.some(version => version.game === game.id)) { items.push({ kind: 'library', label: game.name, state: 'blocked', detail: `Upload and publish ${game.name} game files before this host can be enabled.` }); } } else if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting', detail: 'Upload T5 or T6 game files; hosts download the latest available files automatically.' }); const state = items.some(item => item.state === 'blocked') ? 'blocked' : items.some(item => item.state === 'preparing') ? 'preparing' : items.some(item => item.state === 'waiting') ? 'waiting' : 'ready'; return { state, items }; } export function presentHostPreparation(host: Host): Host { const preparation = hostPreparationStatus(host); if (['needs-runtime', 'needs-game-files'].includes(host.readiness) && preparation.state === 'preparing') { return { ...host, readiness: 'preparing', readinessReasons: preparation.items.filter(item => item.state !== 'ready').map(item => item.detail), preparation }; } return { ...host, preparation }; } function supersedeUndeliveredDownloads(hostId: string): void { const required = new Set(targetLibraries(queuedSelections(hostId)).map(version => version.id)); const pending = db().prepare(`SELECT id,json_extract(payload,'$.libraryVersionId') AS versionId FROM ec_host_operations WHERE host_id=? AND kind='library.download' AND state='pending' AND last_delivered_at IS NULL AND json_extract(payload,'$.automatic')=1`).all(hostId) as { id: string; versionId: string }[]; const now = new Date().toISOString(); for (const operation of pending) { if (required.has(operation.versionId)) continue; // Delivered operations may already be writing files. Only unsubmitted // automatic work can be replaced without consulting the agent journal. db().prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?").run(supersededDownload, now, operation.id); db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)").run(operation.id, now, supersededDownload); db().prepare("UPDATE host_library_caches SET status='missing',error=NULL,bytes_verified=NULL,transfer_progress=NULL,updated_at=? WHERE host_id=? AND version_id=? AND operation_id=? AND status NOT IN ('ready','outdated')") .run(now, hostId, operation.versionId, operation.id); } } /** Agent heartbeats and polls drive durable preparation; no open UI or in-memory worker is needed. */ export function reconcileHostPreparation(hostId: string): HostPreparation { return db().transaction(() => { const host = getHost(hostId); if (!host) throw new Error('Host was not found.'); if (!hostBlocker(host)) { supersedeUndeliveredDownloads(hostId); // Queue only one setup mutation at a time so each capacity check uses // fresh disk inventory. This also respects manually queued downloads. const active = db().prepare(`SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId); const finished = db().prepare(`SELECT updated_at FROM ec_host_operations WHERE host_id=? AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('completed','failed') AND (error IS NULL OR error != ?) ORDER BY updated_at DESC LIMIT 1`).get(hostId, supersededDownload) as { updated_at: string } | undefined; if (!active && (!finished || Date.parse(host.lastHeartbeat!) > Date.parse(finished.updated_at))) { // A failed or oversized library must not prevent other libraries // from preparing. The runtime still comes first when eligible. const target = targets(host).find(target => target.item.state === 'waiting'); if (target?.image) queueHostOperation(host.id, 'runtime.pull', { image: target.image }); else if (target?.versionId) queueLibraryDownload(host.id, target.versionId, { automatic: true }); } } const status = hostPreparationStatus(host); if (host.enablingSince && !hostBlocker(host) && status.state === 'ready') { db().prepare("UPDATE ec_hosts SET mode='active' WHERE id=? AND mode='disabled'").run(hostId); db().prepare('DELETE FROM ec_host_enable_requests WHERE host_id=?').run(hostId); scheduleAvailabilityRefresh(); } return status; }).immediate(); }