josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate instances when draining and verify hosts before enabling

Browse Source
This commit is contained in:
josh
2026-10-05 14:21:48 -04:00
parent a142f06c77
commit 16a5bc3534
20 changed files with 525 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/hosts/availability.ts
+3 -1
View File
@@ -52,7 +52,9 @@ function observeHost(host: Host): HostAvailability {
JOIN native_provisioning n ON n.instance_id=i.id JOIN server_deployments d ON d.id=n.deployment_id
WHERE i.host_id=? AND i.retired_at IS NULL AND d.action IN ('create','rebuild')
AND d.state IN ('running','awaiting-health')`).all(host.id) as { id: string; generation: number }[];
const unmeasured = pending.filter(instance => measured.get(instance.id) !== instance.generation).length;
const incoming = handle.prepare(`SELECT COUNT(*) AS n FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.target_host_id=? AND (m.removed_at IS NULL OR d.state IN ('queued','waiting','interrupted'))`).get(host.id) as { n: number };
const unmeasured = pending.filter(instance => measured.get(instance.id) !== instance.generation).length + incoming.n;
const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
const freeCpu = cpuCount && cpuUsagePercent !== undefined
? Math.max(0, cpuCount * (0.95 - cpuUsagePercent / 100) - unmeasured * startup.cpuCores) : null;
src/lib/server/hosts/capacity.ts
+6 -4
View File
@@ -46,7 +46,7 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
}
const instances = db().prepare(`SELECT i.id, i.generation,
NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE n.instance_id=i.id AND d.action='create' AND d.state IN ('done','warning') AND d.finished_at<?) AS pending,
WHERE n.instance_id=i.id AND n.generation=i.generation AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at<?) AS pending,
EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE n.instance_id=i.id AND d.action='create' AND d.state='queued' AND d.submitted_at IS NULL
AND n.operation_id IS NULL AND n.stage_operation_id IS NULL) AS unsubmitted
@@ -54,13 +54,15 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
.all(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as
{ id: string; generation: number; pending: number; unsubmitted: number }[];
const { running, startup } = capacityCalibration(host.id);
const incoming = db().prepare(`SELECT COUNT(*) AS n FROM ec_host_migrations
WHERE target_host_id=? AND removed_at IS NULL AND instance_id!=?`).get(host.id, options.excludeInstanceId ?? '') as { n: number };
const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
// Admission budgets the eventual running collection. Dispatch budgets only
// the next startup; the rest of the queue waits for live capacity to recover.
const launch = options.dispatch ? startup : running;
let extraCpu = planned * running.cpuCores + launch.cpuCores;
let extraMemory = planned * running.memoryBytes + launch.memoryBytes;
let reservations = planned + 1;
let extraCpu = planned * running.cpuCores + launch.cpuCores + incoming.n * startup.cpuCores;
let extraMemory = planned * running.memoryBytes + launch.memoryBytes + incoming.n * startup.memoryBytes;
let reservations = planned + 1 + incoming.n;
const measurements = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample]));
for (const instance of instances) {
if (options.dispatch && options.excludeInstanceId && instance.unsubmitted) continue;
src/lib/server/hosts/index.ts
+23 -7
View File
@@ -37,6 +37,7 @@ function presentDecommission(row: DecommissionRow | undefined): HostDecommission
error: row.error, retryable: row.stage === 'failed' && getHostOperationStatus(row.operation_id)?.state === 'failed' } : null;
}
function hostFromRow(row: HostRow): Host {
const enabling = database().prepare('SELECT requested_at FROM ec_host_enable_requests WHERE host_id=?').get(row.id) as { requested_at: string } | undefined;
const inventory: HostInventory = JSON.parse(row.inventory);
const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
const reasons: string[] = [];
@@ -53,13 +54,14 @@ function hostFromRow(row: HostRow): Host {
if (!runtimeReady) reasons.push('The Plutainer runtime image is not ready yet.');
if (!libraryReady) reasons.push('No verified game-file library is available yet.');
if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
if (row.mode === 'disabled') reasons.push('Host is disabled.');
if (row.mode === 'disabled') reasons.push(enabling ? 'Host setup must refresh the runtime and verify both BO1 and BO2 game files before enabling.' : 'Host is disabled.');
const decommission = presentDecommission(decommissionRow(row.id));
if (decommission) reasons.push('Host deletion is in progress; waiting for confirmed cleanup and agent removal.');
return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max,
bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes,
createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline',
readiness: decommission ? 'deleting' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
enablingSince: enabling?.requested_at ?? null,
readiness: decommission ? 'deleting' : enabling ? 'preparing' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active, decommission };
}
export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
@@ -197,7 +199,7 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
const host = requireHost(id);
if (host.decommission) throw new Error('Host cleanup is in progress; its settings cannot change.');
const name = input.name === undefined ? host.name : textField(input.name, 'Host name');
const mode = input.mode ?? host.mode;
let mode = input.mode ?? host.mode;
if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.');
const min = port(input.portMin ?? host.portMin); const max = port(input.portMax ?? host.portMax);
if (max < min) throw new Error('The last game port must not precede the first.');
@@ -206,7 +208,20 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
const publicAddress = input.publicAddress === undefined ? host.publicAddress : input.publicAddress === '' ? '' : textField(input.publicAddress, 'Public address');
const minimum = input.minimumFreeBytes ?? host.minimumFreeBytes;
if (typeof minimum !== 'number' || !Number.isSafeInteger(minimum) || minimum < 0) throw new Error('Minimum free storage must be a nonnegative byte count.');
database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id);
database().transaction(() => {
if (input.mode === 'active' && host.mode === 'draining') throw new Error('Wait for this host to finish draining before enabling it.');
if (input.mode === 'active' && host.mode === 'disabled') {
database().prepare('INSERT OR IGNORE INTO ec_host_enable_requests(host_id,requested_at,operation_floor) VALUES(?,?,(SELECT COALESCE(MAX(rowid),0) FROM ec_host_operations))').run(id, now());
mode = 'disabled';
}
if (input.mode === 'draining') {
if (host.enablingSince) throw new Error('Wait for automatic host setup to finish before draining.');
const incoming = database().prepare(`SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.target_host_id=? AND d.state NOT IN ('done','warning') AND d.server_deleted_at IS NULL LIMIT 1`).get(id);
if (incoming) throw new Error('Wait for incoming instance migrations to finish before draining this host.');
}
database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id);
})();
scheduleAvailabilityRefresh();
return requireHost(id);
}
@@ -435,7 +450,7 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu
}
}
if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.');
if (host.mode === 'disabled' && kind !== 'instance.cancel-create' && !(host.enablingSince && ['runtime.pull','library.download','host.inspect'].includes(kind))) throw new Error('Host is disabled. Enable it before requesting operations.');
const id = options.id ?? randomUUID(); const timestamp = now();
const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?')
.get(options.instanceId) as { server: string } | undefined)?.server ?? null : null;
@@ -461,7 +476,7 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane, in
/** Delivery snapshots retain status/results while large immutable payloads stay in SQLite. */
export function pendingHostOperationRows(hostId: string, lane: OperationLane = 'all', includePayload = true): OperationDeliveryRow[] {
const host = requireHost(hostId);
if (host.mode === 'disabled' || !host.connected) return [];
if ((host.mode === 'disabled' && !host.enablingSince) || !host.connected) return [];
// Older agents hold activity.lock across the entire delivered batch, including
// observations. Filtering only the unsupported cleanup can keep the updater
// busy indefinitely. Drain the old batch by withholding ALL further work;
@@ -484,12 +499,13 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
const columns = includePayload ? `${operationStatusColumns},${operationPayloadSql} AS payload` : operationStatusColumns;
const rows = database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
${laneFilter}
${host.mode === 'disabled' ? "AND kind IN ('runtime.pull','library.download','host.inspect')" : ''}
AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
AND (kind!='instance.cancel-create' OR ?)
AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)
.all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationDeliveryRow[];
.all(hostId, Number(host.readiness === 'ready' || host.mode === 'draining'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationDeliveryRow[];
const deliveredAt = now();
for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
return rows;
src/lib/server/hosts/preparation.ts
+34 -11
View File
@@ -4,10 +4,11 @@ import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation } from './index.js'
import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js';
import { GAME_FILE_GAMES } from '../../game-files.js';
import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js';
import { scheduleAvailabilityRefresh } from './availability.js';
const activeStates = ['pending', 'accepted', 'running', 'unknown'];
const supersededDownload = 'Superseded by newer game files before delivery to the host.';
type PreparationOperation = { id: string; state: HostOperationState; updated_at: string; error: string | null };
type PreparationOperation = { id: string; sequence: number; state: HostOperationState; updated_at: string; error: string | null };
type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation };
type QueuedSelection = { libraryId: string; image: string | null };
type LibraryTarget = Pick<ReturnType<typeof latestPublishedLibraries>[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean };
@@ -23,7 +24,12 @@ function queuedSelections(hostId: string): QueuedSelection[] {
WHERE d.action='create' AND d.server_deleted_at IS NULL AND n.deployment_id IS NULL
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
AND (json_extract(q.selection,'$.automatic')=1 OR json_extract(q.selection,'$.hostId') IS NULL
OR json_extract(q.selection,'$.hostId')=?)`).all(hostId) as QueuedSelection[];
OR json_extract(q.selection,'$.hostId')=?)
UNION SELECT i.library_id AS libraryId,i.image FROM ec_host_migrations m
JOIN native_instances i ON i.id=m.instance_id JOIN server_deployments d ON d.id=m.deployment_id
WHERE m.removed_at IS NULL AND d.server_deleted_at IS NULL
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
AND (m.target_host_id IS NULL OR m.target_host_id=?)`).all(hostId, hostId) as QueuedSelection[];
}
function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
@@ -44,7 +50,7 @@ function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
}
function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined {
return db().prepare(`SELECT id,state,updated_at,error FROM ec_host_operations
return db().prepare(`SELECT id,rowid AS sequence,state,updated_at,error FROM ec_host_operations
WHERE host_id=? AND kind=? AND json_extract(payload,?)=? ORDER BY created_at DESC,rowid DESC LIMIT 1`)
.get(hostId, kind, `$.${field}`, value) as PreparationOperation | undefined;
}
@@ -67,7 +73,7 @@ function retryAt(hostId: string, target: Target): number {
function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string } | null {
if (host.decommission) return { state: 'waiting', detail: 'Automatic preparation is paused during host deletion.' };
if (host.mode !== 'active') return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
if (host.mode !== 'active' && !host.enablingSince) return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
if (!host.enrolled) return { state: 'waiting', detail: 'Automatic preparation starts after agent installation.' };
if (!host.connected || !host.credentialActive) return { state: 'waiting', detail: 'Waiting for the host to reconnect.' };
if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) return { state: 'blocked', detail: 'A compatible host agent is required.' };
@@ -77,20 +83,26 @@ function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string
}
function targets(host: Host): Target[] {
const enabling = db().prepare('SELECT operation_floor FROM ec_host_enable_requests WHERE host_id=?').get(host.id) as { operation_floor: number } | undefined;
const verifiedThisEnable = (operation: PreparationOperation | undefined) => !enabling || !!operation
&& operation.sequence > enabling.operation_floor && operation.state === 'completed'
&& Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at);
const selections = queuedSelections(host.id);
const images = new Set([config.komodo.defaultImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]);
const result: Target[] = [...images].map(image => {
const runtimeReady = host.inventory.runtimeImages?.includes(image);
const operation = latestOperation(host.id, 'runtime.pull', 'image', image);
const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
return { image, requiredBytes: host.minimumFreeBytes,
operation: latestOperation(host.id, 'runtime.pull', 'image', image),
operation,
item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } };
});
for (const version of targetLibraries(selections)) {
const cache = getHostLibraryCache(host.id, version.id);
const matches = cache?.manifestSha256 === version.manifestSha256;
const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes;
const operation = latestOperation(host.id, 'library.download', 'libraryVersionId', version.id);
const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes && verifiedThisEnable(operation);
result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
operation: latestOperation(host.id, 'library.download', 'libraryVersionId', version.id),
operation,
item: { kind: 'library', label: version.name, state: ready ? 'ready' : 'waiting',
detail: version.queued
? ready ? 'Game files selected by queued launches are downloaded and verified.' : 'Game files selected by queued launches will be downloaded and verified automatically.'
@@ -110,7 +122,7 @@ function targets(host: Host): Target[] {
: target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.';
} else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) {
target.item.state = 'preparing'; target.item.detail = 'Waiting for a fresh host inventory after preparation.';
} else if (retryAt(host.id, target) > Date.now()) {
} else if ((!enabling || (operation?.sequence ?? 0) > enabling.operation_floor) && retryAt(host.id, target) > Date.now()) {
target.item.state = 'blocked';
target.item.detail = `${operation?.error ?? 'Preparation failed.'} Automatic retry after ${new Date(retryAt(host.id, target)).toISOString()}.`;
} else if (Number(host.inventory.storageAvailable ?? 0) < target.requiredBytes) {
@@ -122,7 +134,12 @@ function targets(host: Host): Target[] {
export function hostPreparationStatus(host: Host): HostPreparation {
const items = targets(host).map(target => target.item);
if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
if (host.enablingSince) {
const published = latestPublishedLibraries();
for (const game of GAME_FILE_GAMES) if (!published.some(version => version.game === game.id)) {
items.push({ kind: 'library', label: game.name, state: 'blocked', detail: `Upload and publish ${game.name} game files before this host can be enabled.` });
}
} else if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
detail: 'Upload T5 or T6 game files; hosts download the latest available files automatically.' });
const state = items.some(item => item.state === 'blocked') ? 'blocked'
: items.some(item => item.state === 'preparing') ? 'preparing'
@@ -178,6 +195,12 @@ export function reconcileHostPreparation(hostId: string): HostPreparation {
else if (target?.versionId) queueLibraryDownload(host.id, target.versionId, { automatic: true });
}
}
return hostPreparationStatus(host);
const status = hostPreparationStatus(host);
if (host.enablingSince && !hostBlocker(host) && status.state === 'ready') {
db().prepare("UPDATE ec_hosts SET mode='active' WHERE id=? AND mode='disabled'").run(hostId);
db().prepare('DELETE FROM ec_host_enable_requests WHERE host_id=?').run(hostId);
scheduleAvailabilityRefresh();
}
return status;
}).immediate();
}
src/lib/server/hosts/schema.ts
+3
View File
@@ -19,6 +19,9 @@ export function ensureHostSchema(handle: Database.Database): void {
id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), token_hash TEXT NOT NULL UNIQUE,
created_at TEXT NOT NULL, expires_at TEXT NOT NULL, consumed_at TEXT
);
CREATE TABLE IF NOT EXISTS ec_host_enable_requests (
host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id), requested_at TEXT NOT NULL, operation_floor INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS ec_host_credentials (
id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), token_hash TEXT NOT NULL UNIQUE,
created_at TEXT NOT NULL, revoked_at TEXT, last_used_at TEXT