josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate instances when draining and verify hosts before enabling

Browse Source
This commit is contained in:
josh
2026-10-05 14:21:48 -04:00
parent a142f06c77
commit 16a5bc3534
20 changed files with 525 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/hosts/preparation.ts
+34 -11
View File
@@ -4,10 +4,11 @@ import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation } from './index.js'
import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js';
import { GAME_FILE_GAMES } from '../../game-files.js';
import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js';
import { scheduleAvailabilityRefresh } from './availability.js';
const activeStates = ['pending', 'accepted', 'running', 'unknown'];
const supersededDownload = 'Superseded by newer game files before delivery to the host.';
type PreparationOperation = { id: string; state: HostOperationState; updated_at: string; error: string | null };
type PreparationOperation = { id: string; sequence: number; state: HostOperationState; updated_at: string; error: string | null };
type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation };
type QueuedSelection = { libraryId: string; image: string | null };
type LibraryTarget = Pick<ReturnType<typeof latestPublishedLibraries>[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean };
@@ -23,7 +24,12 @@ function queuedSelections(hostId: string): QueuedSelection[] {
WHERE d.action='create' AND d.server_deleted_at IS NULL AND n.deployment_id IS NULL
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
AND (json_extract(q.selection,'$.automatic')=1 OR json_extract(q.selection,'$.hostId') IS NULL
OR json_extract(q.selection,'$.hostId')=?)`).all(hostId) as QueuedSelection[];
OR json_extract(q.selection,'$.hostId')=?)
UNION SELECT i.library_id AS libraryId,i.image FROM ec_host_migrations m
JOIN native_instances i ON i.id=m.instance_id JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.removed_at IS NULL AND d.server_deleted_at IS NULL
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
AND (m.target_host_id IS NULL OR m.target_host_id=?)`).all(hostId, hostId) as QueuedSelection[];
}
function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
@@ -44,7 +50,7 @@ function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
}
function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined {
return db().prepare(`SELECT id,state,updated_at,error FROM ec_host_operations
return db().prepare(`SELECT id,rowid AS sequence,state,updated_at,error FROM ec_host_operations
WHERE host_id=? AND kind=? AND json_extract(payload,?)=? ORDER BY created_at DESC,rowid DESC LIMIT 1`)
.get(hostId, kind, `$.${field}`, value) as PreparationOperation | undefined;
}
@@ -67,7 +73,7 @@ function retryAt(hostId: string, target: Target): number {
function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string } | null {
if (host.decommission) return { state: 'waiting', detail: 'Automatic preparation is paused during host deletion.' };
if (host.mode !== 'active') return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
if (host.mode !== 'active' && !host.enablingSince) return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
if (!host.enrolled) return { state: 'waiting', detail: 'Automatic preparation starts after agent installation.' };
if (!host.connected || !host.credentialActive) return { state: 'waiting', detail: 'Waiting for the host to reconnect.' };
if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) return { state: 'blocked', detail: 'A compatible host agent is required.' };
@@ -77,20 +83,26 @@ function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string
}
function targets(host: Host): Target[] {
const enabling = db().prepare('SELECT operation_floor FROM ec_host_enable_requests WHERE host_id=?').get(host.id) as { operation_floor: number } | undefined;
const verifiedThisEnable = (operation: PreparationOperation | undefined) => !enabling || !!operation
&& operation.sequence > enabling.operation_floor && operation.state === 'completed'
&& Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at);
const selections = queuedSelections(host.id);
const images = new Set([config.komodo.defaultImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]);
const result: Target[] = [...images].map(image => {
const runtimeReady = host.inventory.runtimeImages?.includes(image);
const operation = latestOperation(host.id, 'runtime.pull', 'image', image);
const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
return { image, requiredBytes: host.minimumFreeBytes,
operation: latestOperation(host.id, 'runtime.pull', 'image', image),
operation,
item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } };
});
for (const version of targetLibraries(selections)) {
const cache = getHostLibraryCache(host.id, version.id);
const matches = cache?.manifestSha256 === version.manifestSha256;
const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes;
const operation = latestOperation(host.id, 'library.download', 'libraryVersionId', version.id);
const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes && verifiedThisEnable(operation);
result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
operation: latestOperation(host.id, 'library.download', 'libraryVersionId', version.id),
operation,
item: { kind: 'library', label: version.name, state: ready ? 'ready' : 'waiting',
detail: version.queued
? ready ? 'Game files selected by queued launches are downloaded and verified.' : 'Game files selected by queued launches will be downloaded and verified automatically.'
@@ -110,7 +122,7 @@ function targets(host: Host): Target[] {
: target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.';
} else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) {
target.item.state = 'preparing'; target.item.detail = 'Waiting for a fresh host inventory after preparation.';
} else if (retryAt(host.id, target) > Date.now()) {
} else if ((!enabling || (operation?.sequence ?? 0) > enabling.operation_floor) && retryAt(host.id, target) > Date.now()) {
target.item.state = 'blocked';
target.item.detail = `${operation?.error ?? 'Preparation failed.'} Automatic retry after ${new Date(retryAt(host.id, target)).toISOString()}.`;
} else if (Number(host.inventory.storageAvailable ?? 0) < target.requiredBytes) {
@@ -122,7 +134,12 @@ function targets(host: Host): Target[] {
export function hostPreparationStatus(host: Host): HostPreparation {
const items = targets(host).map(target => target.item);
if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
if (host.enablingSince) {
const published = latestPublishedLibraries();
for (const game of GAME_FILE_GAMES) if (!published.some(version => version.game === game.id)) {
items.push({ kind: 'library', label: game.name, state: 'blocked', detail: `Upload and publish ${game.name} game files before this host can be enabled.` });
}
} else if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
detail: 'Upload T5 or T6 game files; hosts download the latest available files automatically.' });
const state = items.some(item => item.state === 'blocked') ? 'blocked'
: items.some(item => item.state === 'preparing') ? 'preparing'
@@ -178,6 +195,12 @@ export function reconcileHostPreparation(hostId: string): HostPreparation {
else if (target?.versionId) queueLibraryDownload(host.id, target.versionId, { automatic: true });
}
}
return hostPreparationStatus(host);
const status = hostPreparationStatus(host);
if (host.enablingSince && !hostBlocker(host) && status.state === 'ready') {
db().prepare("UPDATE ec_hosts SET mode='active' WHERE id=? AND mode='disabled'").run(hostId);
db().prepare('DELETE FROM ec_host_enable_requests WHERE host_id=?').run(hostId);
scheduleAvailabilityRefresh();
}
return status;
}).immediate();
}