207 lines
15 KiB
TypeScript
207 lines
15 KiB
TypeScript
import { db } from '../db/index.js';
|
|
import { config } from '../config.js';
|
|
import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation } from './index.js';
|
|
import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js';
|
|
import { GAME_FILE_GAMES } from '../../game-files.js';
|
|
import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js';
|
|
import { scheduleAvailabilityRefresh } from './availability.js';
|
|
|
|
const activeStates = ['pending', 'accepted', 'running', 'unknown'];
|
|
const supersededDownload = 'Superseded by newer game files before delivery to the host.';
|
|
type PreparationOperation = { id: string; sequence: number; state: HostOperationState; updated_at: string; error: string | null };
|
|
type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation };
|
|
type QueuedSelection = { libraryId: string; image: string | null };
|
|
type LibraryTarget = Pick<ReturnType<typeof latestPublishedLibraries>[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean };
|
|
|
|
/** Accepted requests retain their prerequisites until placement owns an instance. */
|
|
function queuedSelections(hostId: string): QueuedSelection[] {
|
|
// Standalone host/library readers can initialize before provisioning migrations.
|
|
if (!db().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'").get()) return [];
|
|
return db().prepare(`SELECT DISTINCT json_extract(q.selection,'$.libraryId') AS libraryId,
|
|
json_extract(q.selection,'$.image') AS image FROM native_creation_intents q
|
|
JOIN server_deployments d ON d.id=q.deployment_id
|
|
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
|
|
WHERE d.action='create' AND d.server_deleted_at IS NULL AND n.deployment_id IS NULL
|
|
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
|
|
AND (json_extract(q.selection,'$.automatic')=1 OR json_extract(q.selection,'$.hostId') IS NULL
|
|
OR json_extract(q.selection,'$.hostId')=?)
|
|
UNION SELECT i.library_id AS libraryId,i.image FROM ec_host_migrations m
|
|
JOIN native_instances i ON i.id=m.instance_id JOIN server_deployments d ON d.id=m.deployment_id
|
|
WHERE m.removed_at IS NULL AND d.server_deleted_at IS NULL
|
|
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
|
|
AND (m.target_host_id IS NULL OR m.target_host_id=?)`).all(hostId, hostId) as QueuedSelection[];
|
|
}
|
|
|
|
function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
|
|
const latest = latestPublishedLibraries();
|
|
const latestIds = new Set(latest.map(version => version.id));
|
|
const queued = new Map<string, LibraryTarget>();
|
|
const get = db().prepare(`SELECT v.id,l.game,v.version,v.size_bytes AS sizeBytes,v.manifest_sha256 AS manifestSha256
|
|
FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
|
|
WHERE v.id=? AND v.state='ready' AND NOT EXISTS(SELECT 1 FROM game_file_cleanup c WHERE c.version_id=v.id)`);
|
|
for (const selection of selections) {
|
|
if (latestIds.has(selection.libraryId) || queued.has(selection.libraryId)) continue;
|
|
const version = get.get(selection.libraryId) as { id: string; game: string; version: string; sizeBytes: number; manifestSha256: string } | undefined;
|
|
if (version) queued.set(version.id, { ...version, queued: true,
|
|
name: `${GAME_FILE_GAMES.find(game => game.id === version.game)?.name ?? version.game} · Queued version ${version.version}` });
|
|
}
|
|
// Finish prerequisites for accepted work before downloading newer optional files.
|
|
return [...queued.values(), ...latest];
|
|
}
|
|
|
|
function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined {
|
|
return db().prepare(`SELECT id,rowid AS sequence,state,updated_at,error FROM ec_host_operations
|
|
WHERE host_id=? AND kind=? AND json_extract(payload,?)=? ORDER BY created_at DESC,rowid DESC LIMIT 1`)
|
|
.get(hostId, kind, `$.${field}`, value) as PreparationOperation | undefined;
|
|
}
|
|
|
|
function retryAt(hostId: string, target: Target): number {
|
|
const operation = target.operation;
|
|
if (!operation || operation.state !== 'failed') return 0;
|
|
const kind = target.image ? 'runtime.pull' : 'library.download';
|
|
const field = target.image ? '$.image' : '$.libraryVersionId';
|
|
const value = target.image ?? target.versionId!;
|
|
// Count consecutive failures, including across controller restarts. Unknown
|
|
// operations are never retried under a new ID: their journal must reconcile.
|
|
const { count } = db().prepare(`SELECT COUNT(*) AS count FROM ec_host_operations
|
|
WHERE host_id=? AND kind=? AND json_extract(payload,?)=? AND state='failed'
|
|
AND rowid>COALESCE((SELECT MAX(rowid) FROM ec_host_operations
|
|
WHERE host_id=? AND kind=? AND json_extract(payload,?)=? AND state='completed'),0)`)
|
|
.get(hostId, kind, field, value, hostId, kind, field, value) as { count: number };
|
|
return Date.parse(operation.updated_at) + Math.min(15 * 60_000, 60_000 * 2 ** Math.min(4, Math.max(0, count - 1)));
|
|
}
|
|
|
|
function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string } | null {
|
|
if (host.decommission) return { state: 'waiting', detail: 'Automatic preparation is paused during host deletion.' };
|
|
if (host.mode !== 'active' && !host.enablingSince) return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
|
|
if (!host.enrolled) return { state: 'waiting', detail: 'Automatic preparation starts after agent installation.' };
|
|
if (!host.connected || !host.credentialActive) return { state: 'waiting', detail: 'Waiting for the host to reconnect.' };
|
|
if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) return { state: 'blocked', detail: 'A compatible host agent is required.' };
|
|
if (!host.inventory.dockerRunning) return { state: 'blocked', detail: 'Waiting for Docker to become available on the host.' };
|
|
if (!host.inventory.directoriesWritable) return { state: 'blocked', detail: 'The host cannot write its managed directories.' };
|
|
return null;
|
|
}
|
|
|
|
function targets(host: Host): Target[] {
|
|
const enabling = db().prepare('SELECT operation_floor FROM ec_host_enable_requests WHERE host_id=?').get(host.id) as { operation_floor: number } | undefined;
|
|
const verifiedThisEnable = (operation: PreparationOperation | undefined) => !enabling || !!operation
|
|
&& operation.sequence > enabling.operation_floor && operation.state === 'completed'
|
|
&& Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at);
|
|
const selections = queuedSelections(host.id);
|
|
const images = new Set([config.komodo.defaultImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]);
|
|
const result: Target[] = [...images].map(image => {
|
|
const operation = latestOperation(host.id, 'runtime.pull', 'image', image);
|
|
const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
|
|
return { image, requiredBytes: host.minimumFreeBytes,
|
|
operation,
|
|
item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } };
|
|
});
|
|
for (const version of targetLibraries(selections)) {
|
|
const cache = getHostLibraryCache(host.id, version.id);
|
|
const matches = cache?.manifestSha256 === version.manifestSha256;
|
|
const operation = latestOperation(host.id, 'library.download', 'libraryVersionId', version.id);
|
|
const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes && verifiedThisEnable(operation);
|
|
result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
|
|
operation,
|
|
item: { kind: 'library', label: version.name, state: ready ? 'ready' : 'waiting',
|
|
detail: version.queued
|
|
? ready ? 'Game files selected by queued launches are downloaded and verified.' : 'Game files selected by queued launches will be downloaded and verified automatically.'
|
|
: ready ? 'The latest game files are downloaded and verified.' : 'The latest game files will be downloaded and verified automatically.',
|
|
bytesDownloaded: cache?.bytesDownloaded ?? 0, bytesTotal: version.sizeBytes, ...(cache ? { cache } : {}) } });
|
|
}
|
|
const blocker = hostBlocker(host);
|
|
for (const target of result) {
|
|
if (target.item.state === 'ready') continue;
|
|
const operation = target.operation;
|
|
if (operation) target.item.operationId = operation.id;
|
|
if (blocker) { Object.assign(target.item, blocker); continue; }
|
|
if (operation && activeStates.includes(operation.state)) {
|
|
target.item.state = 'preparing';
|
|
target.item.detail = operation.state === 'unknown' ? 'Waiting for the agent to reconcile interrupted preparation.'
|
|
: target.image ? 'Downloading the runtime automatically.'
|
|
: target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.';
|
|
} else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) {
|
|
target.item.state = 'preparing'; target.item.detail = 'Waiting for a fresh host inventory after preparation.';
|
|
} else if ((!enabling || (operation?.sequence ?? 0) > enabling.operation_floor) && retryAt(host.id, target) > Date.now()) {
|
|
target.item.state = 'blocked';
|
|
target.item.detail = `${operation?.error ?? 'Preparation failed.'} Automatic retry after ${new Date(retryAt(host.id, target)).toISOString()}.`;
|
|
} else if (Number(host.inventory.storageAvailable ?? 0) < target.requiredBytes) {
|
|
target.item.state = 'blocked'; target.item.detail = 'Insufficient free storage, including the host reserve. Preparation resumes automatically when space is available.';
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
export function hostPreparationStatus(host: Host): HostPreparation {
|
|
const items = targets(host).map(target => target.item);
|
|
if (host.enablingSince) {
|
|
const published = latestPublishedLibraries();
|
|
for (const game of GAME_FILE_GAMES) if (!published.some(version => version.game === game.id)) {
|
|
items.push({ kind: 'library', label: game.name, state: 'blocked', detail: `Upload and publish ${game.name} game files before this host can be enabled.` });
|
|
}
|
|
} else if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
|
|
detail: 'Upload T5 or T6 game files; hosts download the latest available files automatically.' });
|
|
const state = items.some(item => item.state === 'blocked') ? 'blocked'
|
|
: items.some(item => item.state === 'preparing') ? 'preparing'
|
|
: items.some(item => item.state === 'waiting') ? 'waiting' : 'ready';
|
|
return { state, items };
|
|
}
|
|
|
|
export function presentHostPreparation(host: Host): Host {
|
|
const preparation = hostPreparationStatus(host);
|
|
if (['needs-runtime', 'needs-game-files'].includes(host.readiness) && preparation.state === 'preparing') {
|
|
return { ...host, readiness: 'preparing', readinessReasons: preparation.items.filter(item => item.state !== 'ready').map(item => item.detail), preparation };
|
|
}
|
|
return { ...host, preparation };
|
|
}
|
|
|
|
function supersedeUndeliveredDownloads(hostId: string): void {
|
|
const required = new Set(targetLibraries(queuedSelections(hostId)).map(version => version.id));
|
|
const pending = db().prepare(`SELECT id,json_extract(payload,'$.libraryVersionId') AS versionId
|
|
FROM ec_host_operations WHERE host_id=? AND kind='library.download' AND state='pending'
|
|
AND last_delivered_at IS NULL AND json_extract(payload,'$.automatic')=1`).all(hostId) as { id: string; versionId: string }[];
|
|
const now = new Date().toISOString();
|
|
for (const operation of pending) {
|
|
if (required.has(operation.versionId)) continue;
|
|
// Delivered operations may already be writing files. Only unsubmitted
|
|
// automatic work can be replaced without consulting the agent journal.
|
|
db().prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?").run(supersededDownload, now, operation.id);
|
|
db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)").run(operation.id, now, supersededDownload);
|
|
db().prepare("UPDATE host_library_caches SET status='missing',error=NULL,bytes_verified=NULL,transfer_progress=NULL,updated_at=? WHERE host_id=? AND version_id=? AND operation_id=? AND status NOT IN ('ready','outdated')")
|
|
.run(now, hostId, operation.versionId, operation.id);
|
|
}
|
|
}
|
|
|
|
/** Agent heartbeats and polls drive durable preparation; no open UI or in-memory worker is needed. */
|
|
export function reconcileHostPreparation(hostId: string): HostPreparation {
|
|
return db().transaction(() => {
|
|
const host = getHost(hostId);
|
|
if (!host) throw new Error('Host was not found.');
|
|
if (!hostBlocker(host)) {
|
|
supersedeUndeliveredDownloads(hostId);
|
|
// Queue only one setup mutation at a time so each capacity check uses
|
|
// fresh disk inventory. This also respects manually queued downloads.
|
|
const active = db().prepare(`SELECT 1 FROM ec_host_operations WHERE host_id=?
|
|
AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('pending','accepted','running','unknown') LIMIT 1`).get(hostId);
|
|
const finished = db().prepare(`SELECT updated_at FROM ec_host_operations WHERE host_id=?
|
|
AND kind IN ('runtime.pull','library.download','library.import') AND state IN ('completed','failed')
|
|
AND (error IS NULL OR error != ?)
|
|
ORDER BY updated_at DESC LIMIT 1`).get(hostId, supersededDownload) as { updated_at: string } | undefined;
|
|
if (!active && (!finished || Date.parse(host.lastHeartbeat!) > Date.parse(finished.updated_at))) {
|
|
// A failed or oversized library must not prevent other libraries
|
|
// from preparing. The runtime still comes first when eligible.
|
|
const target = targets(host).find(target => target.item.state === 'waiting');
|
|
if (target?.image) queueHostOperation(host.id, 'runtime.pull', { image: target.image });
|
|
else if (target?.versionId) queueLibraryDownload(host.id, target.versionId, { automatic: true });
|
|
}
|
|
}
|
|
const status = hostPreparationStatus(host);
|
|
if (host.enablingSince && !hostBlocker(host) && status.state === 'ready') {
|
|
db().prepare("UPDATE ec_hosts SET mode='active' WHERE id=? AND mode='disabled'").run(hostId);
|
|
db().prepare('DELETE FROM ec_host_enable_requests WHERE host_id=?').run(hostId);
|
|
scheduleAvailabilityRefresh();
|
|
}
|
|
return status;
|
|
}).immediate();
|
|
}
|