import { packFileJson, unpackFileJson, referenceBlobs } from '../db/file-blobs.js'; import { pruneHistory } from '../retention.js'; import { recordAgentReceipt } from '../provisioning/agent-events.js'; import { EventEmitter } from 'node:events'; import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'; import { isIP } from 'node:net'; import { db } from '../db/index.js'; import { ensureHostSchema } from './schema.js'; import { successfulObservation, routineObservationSql, importantOperationSql, operationTargetSql, safeOperationTarget } from './observations.js'; import { currentOperationSource } from './operation-context.js'; import { parseHostInventory } from './request-context.js'; import { fleetSettings } from '../settings.js'; import { recordInstanceResources, validateInstanceResources } from './calibration.js'; import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js'; import { hostAgentLaneLabel, hostAgentLaneState } from '../../hosts.js'; import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationFilter, HostOperationHistoryPage, HostOperationScope, HostOperationState } from '../../hosts.js'; export const AGENT_PROTOCOL_VERSION = 1; export const HEARTBEAT_MAX_AGE_MS = 90_000; const now = () => new Date().toISOString(); const workEvents = new EventEmitter(); workEvents.setMaxListeners(0); export function waitForHostWork(hostId: string, signal: AbortSignal, timeoutMs = 20_000): Promise { return new Promise(resolve => { const done = () => { clearTimeout(timer); workEvents.off(hostId, done); signal.removeEventListener('abort', done); resolve(); }; const timer = setTimeout(done, timeoutMs); workEvents.on(hostId, done); signal.addEventListener('abort', done, { once: true }); if (signal.aborted) done(); }); } const digest = (token: string) => createHash('sha256').update(token).digest('hex'); function database() { const handle = db(); ensureHostSchema(handle); return handle; } type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number }; const hostSql = `SELECT h.*, EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id = h.id AND c.revoked_at IS NULL) AS credential_active FROM ec_hosts h`; type DecommissionRow = { host_id: string; operation_id: string; token_hash: string; stage: HostDecommission['stage']; requested_at: string; updated_at: string; cleaned_at: string | null; error: string | null }; function decommissionRow(hostId: string): DecommissionRow | undefined { return database().prepare('SELECT * FROM ec_host_decommissions WHERE host_id=?').get(hostId) as DecommissionRow | undefined; } function presentDecommission(row: DecommissionRow | undefined): HostDecommission | null { return row ? { operationId: row.operation_id, stage: row.stage, requestedAt: row.requested_at, updatedAt: row.updated_at, error: row.error, retryable: row.stage === 'failed' && getHostOperationStatus(row.operation_id)?.state === 'failed' } : null; } function hostFromRow(row: HostRow): Host { const enabling = database().prepare('SELECT requested_at FROM ec_host_enable_requests WHERE host_id=?').get(row.id) as { requested_at: string } | undefined; const inventory = parseHostInventory(row.id, row.inventory); const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS; const reasons: string[] = []; if (!row.enrolled_at) reasons.push('Agent has not enrolled.'); else if (!row.credential_active) reasons.push('Agent credential has been revoked.'); if (!connected) reasons.push('Agent heartbeat is unavailable or stale.'); if (connected) for (const lane of inventory.agentLanes ?? []) { const state = hostAgentLaneState(lane); if (state === 'stopped' || state === 'stalled') reasons.push(`Agent ${hostAgentLaneLabel(lane.name).toLowerCase()} worker is ${state}. Check the host agent service.`); } if (inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) reasons.push('A compatible agent protocol is required.'); if (!inventory.dockerRunning) reasons.push('Docker is not ready.'); if (!inventory.directoriesWritable) reasons.push('Managed directories are not writable.'); if (typeof inventory.storageAvailable !== 'number' || inventory.storageAvailable < row.minimum_free_bytes) reasons.push('Insufficient available game storage.'); const infrastructureReady = reasons.length === 0; const runtimeReady = !!inventory.runtimeImages?.length; const libraryReady = !!inventory.libraries?.some(library => library.state === 'ready'); if (!runtimeReady) reasons.push('The Plutainer runtime image is not ready yet.'); if (!libraryReady) reasons.push('No verified game-file library is available yet.'); if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.'); if (row.mode === 'disabled') reasons.push(enabling ? 'Host setup must refresh the runtime and verify both BO1 and BO2 game files before enabling.' : 'Host is disabled.'); const decommission = presentDecommission(decommissionRow(row.id)); if (decommission) reasons.push('Host deletion is in progress; waiting for confirmed cleanup and agent removal.'); return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max, bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes, createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline', enablingSince: enabling?.requested_at ?? null, readiness: decommission ? 'deleting' : enabling ? 'preparing' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready', readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active, decommission }; } export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); } export function getHost(id: string): Host | null { const row = database().prepare(hostSql + ' WHERE h.id = ? AND h.deleted_at IS NULL').get(id) as HostRow | undefined; return row ? hostFromRow(row) : null; } function requireHost(id: string): Host { const host = getHost(id); if (!host) throw new Error('Host was not found.'); return host; } export class HostDeletionError extends Error { constructor(message: string, readonly status: 400 | 401 | 404 | 409, readonly blockers: string[] = []) { super(message); } } /** Controller records can be removed only after all remote instance ownership is resolved. */ export function hostDeletionStatus(id: string): HostDeletion { const host = getHost(id); if (!host) throw new HostDeletionError('Host not found.', 404); if (host.decommission && !host.decommission.retryable) return { allowed: true, blockers: [] }; const handle = database(); const blockers: string[] = []; if (host.enrolled && !host.inventory.capabilities?.includes('host.decommission')) blockers.push('Update this host agent before deleting it: generate an installer from Agent access, rerun it on this host, and wait for a fresh heartbeat.'); if (host.enrolled && !host.credentialActive) blockers.push('Re-enroll this host so its agent can receive and complete the cleanup request.'); const instances = handle.prepare('SELECT server FROM native_instances WHERE host_id=? AND retired_at IS NULL ORDER BY server').all(id) as { server: string }[]; if (instances.length) { const names = instances.slice(0, 5).map(instance => instance.server).join(', ') + (instances.length > 5 ? ', …' : ''); blockers.push(`Remove the ${instances.length} server instance${instances.length === 1 ? '' : 's'} assigned to this host first (${names}). Queued and failed deployments also require removal.`); } const pinned = handle.prepare(`SELECT d.server FROM server_deployments d JOIN native_creation_intents q ON q.deployment_id=d.id LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.action='create' AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND n.deployment_id IS NULL AND json_extract(q.selection,'$.hostId')=? AND COALESCE(json_extract(q.selection,'$.automatic'),0)!=1 ORDER BY d.server LIMIT 6`).all(id) as { server: string }[]; if (pinned.length) { const names = pinned.slice(0, 5).map(job => job.server).join(', ') + (pinned.length > 5 ? ', …' : ''); blockers.push(`Cancel the queued server launches pinned to this host before deleting it (${names}).`); } const ports = handle.prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE host_id=?').get(id) as { count: number }; if (ports.count && !instances.length) blockers.push(`This host still has ${ports.count} reserved game port${ports.count === 1 ? '' : 's'}. Finish or reconcile the corresponding server removals first.`); // A pending command may have reached the agent before its acknowledgement. // Only commands that have never been delivered are safe to cancel locally. const operations = handle.prepare(`SELECT kind,state FROM ec_host_operations WHERE host_id=? AND (state IN ('accepted','running','unknown') OR (state='pending' AND last_delivered_at IS NOT NULL)) ORDER BY created_at`).all(id) as { kind: string; state: string }[]; if (operations.length) { const work = operations.slice(0, 5).map(operation => `${operation.kind}: ${operation.state}`).join(', ') + (operations.length > 5 ? ', …' : ''); blockers.push(`${operations.length} delivered operation${operations.length === 1 ? ' is' : 's are'} unresolved (${work}). Let the agent reconnect and finish or reconcile this work before deleting the host.`); } return { allowed: blockers.length === 0, blockers }; } /** Keep the host visible and authenticated until its independent cleanup worker proves removal. */ export function deleteHost(id: string): { deleted: boolean; operationId?: string } { const handle = database(); return handle.transaction(() => { const host = getHost(id); if (!host) throw new HostDeletionError('Host not found.', 404); if (host.decommission && !host.decommission.retryable) return { deleted: false, operationId: host.decommission.operationId }; const status = hostDeletionStatus(id); if (!status.allowed) throw new HostDeletionError(status.blockers.join(' '), 409, status.blockers); const timestamp = now(); const reason = 'Host deletion requested before this operation was delivered to the agent.'; handle.prepare(`INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) SELECT id,'failed',?,? FROM ec_host_operations WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL`).run(timestamp, reason, id); handle.prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL").run(reason, timestamp, id); handle.prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, id); if (!host.enrolled) { handle.prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL").run(timestamp, id); return { deleted: true }; } const operationId = randomUUID(); const token = 'ec_cleanup_' + randomBytes(32).toString('base64url'); const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` }; handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at,source) VALUES(?,?,'host.decommission',?,'pending',?,?,?)") .run(operationId, id, JSON.stringify(payload), timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200)); handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp); handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?) ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`) .run(id, operationId, digest(token), timestamp, timestamp); handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id); queueMicrotask(() => workEvents.emit(id)); return { deleted: false, operationId }; }).immediate(); } export function receiveDecommissionReceipt(operationId: string, token: string, input: Record): { acknowledged: true } { if (!token || token.length > 200) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401); return database().transaction(() => { const row = database().prepare('SELECT * FROM ec_host_decommissions WHERE operation_id=?').get(operationId) as DecommissionRow | undefined; if (!row || !timingSafeEqual(Buffer.from(row.token_hash, 'hex'), Buffer.from(digest(token), 'hex'))) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401); if (!['cleaned', 'completed', 'failed'].includes(input.stage as string)) throw new HostDeletionError('Invalid cleanup receipt stage.', 400); if (row.stage === 'completed') return { acknowledged: true as const }; const operation = getHostOperationStatus(operationId)!; if (!['accepted','running','unknown'].includes(operation.state)) throw new HostDeletionError('Cleanup has not been accepted by the host.', 409); const timestamp = now(); if (input.stage === 'failed') { database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE host_id=?") .run(typeof input.error === 'string' ? input.error.slice(0, 2000) : 'Host cleanup needs attention. The cleanup worker will retry.', timestamp, row.host_id); return { acknowledged: true as const }; } if (input.stage === 'cleaned') { database().prepare("UPDATE ec_host_decommissions SET stage='cleaned',cleaned_at=COALESCE(cleaned_at,?),error=NULL,updated_at=? WHERE host_id=?").run(timestamp, timestamp, row.host_id); return { acknowledged: true as const }; } if (!row.cleaned_at) throw new HostDeletionError('The host must confirm file cleanup before confirming agent removal.', 409); database().prepare("UPDATE ec_host_decommissions SET stage='completed',error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id); database().prepare("UPDATE ec_host_operations SET state='completed',result=?,error=NULL,updated_at=? WHERE id=?").run(JSON.stringify({ filesRemoved: true, agentRemoved: true }), timestamp, operationId); database().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'completed',?)").run(operationId, timestamp); database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(timestamp, row.host_id); database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, row.host_id); if (database().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='host_library_caches'").get()) { database().prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id); } database().prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=?").run(timestamp, row.host_id); return { acknowledged: true as const }; }).immediate(); } function textField(value: unknown, label: string, max = 200): string { if (typeof value !== 'string' || !value.trim() || value.length > max || /[\x00-\x1f]/.test(value)) throw new Error(`${label} must be a nonempty string up to ${max} characters.`); return value.trim(); } function port(value: unknown): number { if (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > 65535) throw new Error('Ports must be whole numbers between 1 and 65535.'); return value; } export function createHost(input: Record): Host { const id = randomUUID(); const name = textField(input.name, 'Host name'); const min = port(input.portMin ?? 28960); const max = port(input.portMax ?? 28980); if (max < min) throw new Error('The last game port must not precede the first.'); const bind = input.bindAddress === undefined ? '0.0.0.0' : textField(input.bindAddress, 'Bind address'); if (!isIP(bind)) throw new Error('Bind address must be an IPv4 or IPv6 address.'); database().prepare('INSERT INTO ec_hosts(id,name,port_min,port_max,bind_address,created_at) VALUES(?,?,?,?,?,?)').run(id, name, min, max, bind, now()); scheduleAvailabilityRefresh(); return requireHost(id); } export function updateHost(id: string, input: Record): Host { const host = requireHost(id); if (host.decommission) throw new Error('Host cleanup is in progress; its settings cannot change.'); const name = input.name === undefined ? host.name : textField(input.name, 'Host name'); let mode = input.mode ?? host.mode; if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.'); const min = port(input.portMin ?? host.portMin); const max = port(input.portMax ?? host.portMax); if (max < min) throw new Error('The last game port must not precede the first.'); const bind = input.bindAddress === undefined ? host.bindAddress : textField(input.bindAddress, 'Bind address'); if (!isIP(bind)) throw new Error('Bind address must be an IPv4 or IPv6 address.'); const publicAddress = input.publicAddress === undefined ? host.publicAddress : input.publicAddress === '' ? '' : textField(input.publicAddress, 'Public address'); const minimum = input.minimumFreeBytes ?? host.minimumFreeBytes; if (typeof minimum !== 'number' || !Number.isSafeInteger(minimum) || minimum < 0) throw new Error('Minimum free storage must be a nonnegative byte count.'); database().transaction(() => { if (input.mode === 'active' && host.mode === 'draining') throw new Error('Wait for this host to finish draining before enabling it.'); if (input.mode === 'active' && host.mode === 'disabled') { database().prepare('INSERT OR IGNORE INTO ec_host_enable_requests(host_id,requested_at,operation_floor) VALUES(?,?,(SELECT COALESCE(MAX(rowid),0) FROM ec_host_operations))').run(id, now()); mode = 'disabled'; } if (input.mode === 'draining') { if (host.enablingSince) throw new Error('Wait for automatic host setup to finish before draining.'); const incoming = database().prepare(`SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id WHERE m.target_host_id=? AND d.state NOT IN ('done','warning') AND d.server_deleted_at IS NULL LIMIT 1`).get(id); if (incoming) throw new Error('Wait for incoming instance migrations to finish before draining this host.'); } database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id); })(); scheduleAvailabilityRefresh(); return requireHost(id); } export function createEnrollment(hostId: string, ttlMs = 15 * 60_000): { id: string; hostId: string; token: string; expiresAt: string } { const host = requireHost(hostId); if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup is in progress; enrollment is unavailable.'); if (!Number.isInteger(ttlMs) || ttlMs <= 0 || ttlMs > 60 * 60_000) throw new Error('Enrollment lifetime must be at most one hour.'); const token = 'ec_enroll_' + randomBytes(32).toString('base64url'); const id = randomUUID(); const expiresAt = new Date(Date.now() + ttlMs).toISOString(); database().transaction(() => { database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(now(), hostId); database().prepare('INSERT INTO ec_host_enrollments(id,host_id,token_hash,created_at,expires_at) VALUES(?,?,?,?,?)').run(id, hostId, digest(token), now(), expiresAt); })(); return { id, hostId, token, expiresAt }; } function issueCredential(hostId: string, credential = 'ec_host_' + randomBytes(32).toString('base64url')): string { database().prepare('INSERT INTO ec_host_credentials(id,host_id,token_hash,created_at) VALUES(?,?,?,?)').run(randomUUID(), hostId, digest(credential), now()); return credential; } export function validateInventory(value: unknown): HostInventory { if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Host inventory must be an object.'); const inventory = value as HostInventory; if (JSON.stringify(inventory).length > 256_000) throw new Error('Host inventory is too large.'); for (const key of ['hostname','agentVersion','os','architecture','cpu','dockerVersion']) { if (inventory[key] !== undefined && (typeof inventory[key] !== 'string' || String(inventory[key]).length > 1000)) throw new Error(`Invalid inventory ${key}.`); } for (const key of ['cpuCount','memoryTotal','memoryAvailable','storageTotal','storageAvailable','protocolVersion']) { if (inventory[key] !== undefined && (typeof inventory[key] !== 'number' || !Number.isSafeInteger(inventory[key]) || Number(inventory[key]) < 0)) throw new Error(`Invalid inventory ${key}.`); } if (inventory.cpuUsagePercent !== undefined && (typeof inventory.cpuUsagePercent !== 'number' || !Number.isFinite(inventory.cpuUsagePercent) || inventory.cpuUsagePercent < 0 || inventory.cpuUsagePercent > 100)) throw new Error('Invalid inventory cpuUsagePercent.'); validateInstanceResources(inventory); if (inventory.instanceStates !== undefined) { if (!Array.isArray(inventory.instanceStates) || inventory.instanceStates.length > 1000) { throw new Error('Invalid instance state inventory.'); } // A malformed server observation must not discard the host heartbeat. inventory.instanceStates = inventory.instanceStates.filter(state => state && typeof state === 'object' && typeof state.instanceId === 'string' && /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(state.instanceId) && Number.isSafeInteger(state.generation) && state.generation >= 1 && typeof state.state === 'string' && /^[a-z-]{1,40}$/.test(state.state) && typeof state.healthy === 'boolean' && (!state.healthy || state.state === 'running') && typeof state.observedAt === 'string' && state.observedAt.length <= 40 && Number.isFinite(Date.parse(state.observedAt))); } for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`); if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.'); if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.'); if (inventory.plutoniumCache !== undefined) { const cache = inventory.plutoniumCache; if (!cache || typeof cache !== 'object' || Array.isArray(cache) || !Number.isSafeInteger(cache.revision) || cache.revision < 1 || !Number.isSafeInteger(cache.bytes) || cache.bytes < 0) throw new Error('Invalid Plutonium cache inventory.'); inventory.plutoniumCache = { revision: cache.revision, bytes: cache.bytes }; } if (inventory.agentUpdate !== undefined) { const update = inventory.agentUpdate; const version = (value: unknown) => typeof value === 'string' && value.length <= 80 && /^\d+\.\d+\.\d+$/.test(value); if (!update || typeof update !== 'object' || Array.isArray(update) || typeof update.enabled !== 'boolean' || !['current','checking','deferred','updating','updated','failed'].includes(update.state) || !version(update.currentVersion) || (update.targetVersion !== undefined && !version(update.targetVersion)) || (update.error !== undefined && (typeof update.error !== 'string' || update.error.length > 2000)) || (update.checkedAt !== undefined && (typeof update.checkedAt !== 'string' || update.checkedAt.length > 80 || !Number.isFinite(Date.parse(update.checkedAt))))) { throw new Error('Invalid agent update inventory.'); } // Only expose display fields, never the local recovery journal or secrets. inventory.agentUpdate = { enabled: update.enabled, state: update.state, currentVersion: update.currentVersion, ...(update.targetVersion === undefined ? {} : { targetVersion: update.targetVersion }), ...(update.error === undefined ? {} : { error: update.error }), ...(update.checkedAt === undefined ? {} : { checkedAt: update.checkedAt }) }; } if (inventory.agentLanes !== undefined) { if (!Array.isArray(inventory.agentLanes) || inventory.agentLanes.length > 4) throw new Error('Invalid agent lane inventory.'); // Keep valid lane diagnostics without discarding a host heartbeat over // one malformed sample. Do not persist thread details or raw errors. const names = new Set(); inventory.agentLanes = inventory.agentLanes.filter(lane => { if (!lane || !['health-1','health-2','read','write'].includes(lane.name) || names.has(lane.name) || typeof lane.alive !== 'boolean' || typeof lane.busy !== 'boolean' || typeof lane.pollAgeSeconds !== 'number' || !Number.isFinite(lane.pollAgeSeconds) || lane.pollAgeSeconds < 0 || (lane.lastPollAt !== null && (typeof lane.lastPollAt !== 'string' || lane.lastPollAt.length > 40 || !Number.isFinite(Date.parse(lane.lastPollAt))))) return false; names.add(lane.name); return true; }).map(({name,alive,busy,lastPollAt,pollAgeSeconds}) => ({name,alive,busy,lastPollAt,pollAgeSeconds})); } if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library => !library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.'); return inventory; } export function enrollHost(token: unknown, rawInventory: unknown): { hostId: string; credential: string; heartbeatSeconds: number } { if (typeof token !== 'string' || token.length > 200) throw new Error('Enrollment token is invalid or expired.'); const inventory = validateInventory(rawInventory); return database().transaction(() => { const enrollment = database().prepare('SELECT id,host_id FROM ec_host_enrollments WHERE token_hash=? AND consumed_at IS NULL AND expires_at>?').get(digest(token), now()) as { id: string; host_id: string } | undefined; if (!enrollment) throw new Error('Enrollment token is invalid or expired.'); const changed = database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE id=? AND consumed_at IS NULL').run(now(), enrollment.id); if (changed.changes !== 1) throw new Error('Enrollment token has already been consumed.'); revokeCredentials(enrollment.host_id); const credential = issueCredential(enrollment.host_id); database().prepare('UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), now(), inventory.hostname ?? '', JSON.stringify(inventory), enrollment.host_id); recordResources(enrollment.host_id, inventory, now()); scheduleAvailabilityRefresh(); return { hostId: enrollment.host_id, credential, heartbeatSeconds: 15 }; })(); } export function authenticateCredential(token: string): Host | null { if (!token || token.length > 200) return null; const row = database().prepare('SELECT id,host_id,last_used_at FROM ec_host_credentials WHERE token_hash=? AND revoked_at IS NULL') .get(digest(token)) as { id: string; host_id: string; last_used_at: string | null } | undefined; if (!row) return null; const timestamp = Date.now(); if (row.last_used_at === null || timestamp - Date.parse(row.last_used_at) >= 60_000) { database().prepare(`UPDATE ec_host_credentials SET last_used_at=? WHERE id=? AND revoked_at IS NULL AND (last_used_at IS NULL OR last_used_at<=?)`) .run(new Date(timestamp).toISOString(), row.id, new Date(timestamp - 60_000).toISOString()); } return getHost(row.host_id); } export function revokeCredentials(hostId: string): void { const host = requireHost(hostId); if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup needs its credentials until removal is confirmed.'); database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(now(), hostId); scheduleAvailabilityRefresh(); } export function rotateCredential(hostId: string, nextCredential?: string): string { requireHost(hostId); if (nextCredential !== undefined && !/^ec_host_[A-Za-z0-9_-]{43}$/.test(nextCredential)) throw new Error('New credentials must contain 32 random bytes encoded as base64url.'); return database().transaction(() => { if (nextCredential) { const existing = database().prepare('SELECT host_id,revoked_at FROM ec_host_credentials WHERE token_hash=?').get(digest(nextCredential)) as { host_id: string; revoked_at: string | null } | undefined; if (existing) { if (existing.host_id !== hostId || existing.revoked_at) throw new Error('This credential cannot be reused.'); return nextCredential; } } revokeCredentials(hostId); return issueCredential(hostId, nextCredential); })(); } export function heartbeat(hostId: string, value: unknown): Host { requireHost(hostId); const inventory = validateInventory(value); database().transaction(() => { const timestamp = now(); database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(timestamp, inventory.hostname ?? '', JSON.stringify(inventory), hostId); recordResources(hostId, inventory, timestamp); })(); const host = requireHost(hostId); refreshHostAvailability(host); return host; } function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void { recordInstanceResources(hostId, inventory, timestamp); database().prepare(`INSERT OR REPLACE INTO ec_host_resource_samples (host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,?,?,?,?)`) .run(hostId, timestamp, inventory.cpuCount ?? null, inventory.cpuUsagePercent ?? null, inventory.memoryTotal ?? null, inventory.memoryAvailable ?? null); if (fleetSettings().hostResourceKeepDays) database().prepare('DELETE FROM ec_host_resource_samples WHERE host_id=? AND sampled_at library.id === options.libraryId && library.state === 'ready')) reasons.push('The selected game library is not ready on this host.'); if (options.image && !host.inventory.runtimeImages?.includes(options.image)) reasons.push('The selected runtime image is not cached on this host.'); if (options.port !== undefined && !hostPortAvailable(hostId, options.port, options.bindAddress, options.protocol)) reasons.push('The selected host port is not available.'); if (reasons.length) throw new Error(reasons.join(' ')); return host; } export type OperationRow = { id: string; host_id: string; kind: string; instance_id: string | null; generation: number | null; payload: string; state: HostOperationState; result: string | null; error: string | null; created_at: string; updated_at: string }; export type OperationDeliveryRow = Omit; export type HostOperationStatus = Omit; const operationStatusColumns = 'o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.result,o.error,o.created_at,o.updated_at'; const operationPayloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)'; const readOnlyOperationSql = `(o.kind IN ('host.inspect','instance.inspect','rcon.status-batch','files.read','files.list','files.stat','files.manifest','logs.read') OR (o.kind='rcon.execute' AND COALESCE(json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version'),0)))`; /** Optional cache downloads yield their write lane to deployments and host changes. */ export function shouldDeferPlutoniumPreparation(hostId: string): boolean { return !!database().prepare(`SELECT 1 FROM ec_host_operations o WHERE o.host_id=? AND o.state IN ('pending','accepted','running','unknown') AND o.kind!='plutonium.prepare' AND NOT ${readOnlyOperationSql} LIMIT 1`).get(hostId); } function operationStatusFromRow(row: Omit): HostOperationStatus { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; } function operationFromRow(row: OperationRow): HostOperation { const status = operationStatusFromRow(row); return { ...status, result: unpackFileJson(status.result), payload: unpackFileJson(JSON.parse(row.payload)) }; } export function getHostOperation(id: string): HostOperation | null { const row = database().prepare(`SELECT ${operationStatusColumns},${operationPayloadSql} AS payload FROM ec_host_operations o WHERE o.id=?`) .get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; } /** Polling and receipts need the outcome, never another copy of launch files. */ export function getHostOperationStatus(id: string): HostOperationStatus | null { const row = database().prepare('SELECT id,host_id,kind,instance_id,generation,state,result,error,created_at,updated_at FROM ec_host_operations WHERE id=?') .get(id) as Omit | undefined; return row ? operationStatusFromRow(row) : null; } export function listHostOperations(hostId: string, limit = 100, summary = false): HostOperation[] { const columns = summary ? `o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.error,o.created_at,o.updated_at,'{}' AS payload,NULL AS result` : `${operationStatusColumns},${operationPayloadSql} AS payload`; return (database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE o.host_id=? AND o.history_expired_at IS NULL ORDER BY o.created_at DESC,o.rowid DESC LIMIT ?`) .all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); } /** Browse retained history without loading command payloads or credential-bearing results. */ export function listHostOperationHistory(page = 1, filter: HostOperationFilter = 'important', scope: HostOperationScope = {}): HostOperationHistoryPage { const handle = database(); const pageSize = 25; const conditions = filter === 'all' ? [] : [filter === 'routine' ? 'o.routine=1' : `(${importantOperationSql})`]; conditions.push('o.history_expired_at IS NULL'); const parameters: string[] = []; if (scope.host) { conditions.push('o.host_id=?'); parameters.push(scope.host); } if (scope.server) { conditions.push('o.server_name=?'); parameters.push(scope.server); } if (scope.operation) { conditions.push('o.id=?'); parameters.push(scope.operation); } const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : ''; return handle.transaction((): HostOperationHistoryPage => { const countColumn = filter === 'all' ? 'total' : filter; // An exact operation seeks its unique ID instead of paging through history. const total = scope.operation ? (handle.prepare(`SELECT 1 AS total FROM ec_host_operations o ${where}`).get(...parameters) as { total: number } | undefined)?.total ?? 0 : (handle.prepare(`SELECT ${countColumn} AS total FROM ec_host_operation_history_counts WHERE host_id=? AND server_name=?`) .get(scope.host ?? '', scope.server ?? '') as { total: number } | undefined)?.total ?? 0; const totalPages = Math.max(1, Math.ceil(total / pageSize)); const currentPage = Math.min(totalPages, Number.isSafeInteger(page) && page > 0 ? page : 1); const operations = handle.prepare(`SELECT o.id,o.host_id AS hostId,h.name AS hostName,o.kind, o.instance_id AS instanceId,o.generation,o.state, CASE WHEN o.routine=1 AND o.state='completed' AND o.observation_success=0 THEN COALESCE(NULLIF(o.error,''),'Observation reported a problem') ELSE o.error END AS error, o.created_at AS createdAt,o.updated_at AS updatedAt,o.routine,o.server_name AS serverName,o.target_path AS targetPath,o.source FROM ec_host_operations o LEFT JOIN ec_hosts h ON h.id=o.host_id AND h.deleted_at IS NULL ${where} ORDER BY o.created_at DESC,o.rowid DESC LIMIT ? OFFSET ?`) .all(...parameters, pageSize, (currentPage - 1) * pageSize) as HostOperationHistoryPage['operations']; return { operations: operations.map(operation => ({ ...operation, routine: !!operation.routine, targetPath: safeOperationTarget(operation.kind, operation.targetPath) })), filter, page: currentPage, pageSize, total, totalPages }; })(); } /** Bound each cleanup pass so retained history cannot stall monitoring or receipts. */ export function pruneHostObservations(): number { return pruneHistory('observation'); } /** Heartbeats stop either launch generation at its next safe command boundary. */ export function pendingCreationCancellations(hostId: string): string[] { const rows = database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS creation_id, json_extract(payload,'$.recoveryRebuildOperationId') AS recovery_id FROM ec_host_operations WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`) .all(hostId) as { creation_id: string; recovery_id: string | null }[]; return rows.flatMap(row => row.recovery_id ? [row.creation_id, row.recovery_id] : [row.creation_id]); } type QueueHostOperationOptions = { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string }; export function queueHostOperation(hostId: string, kind: string, payload: Record, options: QueueHostOperationOptions = {}): HostOperation { if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.'); const operation = enqueueHostOperation(hostId, kind, JSON.stringify(payload), options); return getHostOperation(operation.id)!; } /** Keep immutable launch bytes separate so receipts and delivery do not rewrite large payloads. */ export function queueHostOperationSerialized(hostId: string, kind: string, payload: Buffer, options: QueueHostOperationOptions = {}, routingPayload: Record = {}): HostOperationStatus { if (!Buffer.isBuffer(payload) || payload[0] !== 123 || payload[payload.length - 1] !== 125) throw new Error('Serialized operation payload must be a JSON object.'); if (!routingPayload || typeof routingPayload !== 'object' || Array.isArray(routingPayload)) throw new Error('Operation routing payload must be an object.'); return enqueueHostOperation(hostId, kind, payload, options, JSON.stringify(routingPayload)); } function enqueueHostOperation(hostId: string, kind: string, payload: string | Buffer, options: QueueHostOperationOptions, routingPayload?: string): HostOperationStatus { const host = requireHost(hostId); if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.'); if (kind === 'host.decommission') throw new Error('Use the host deletion workflow to request cleanup.'); if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.'); if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.'); const operation = database().transaction(() => { if (options.idempotencyKey) { const existing = database().prepare(`SELECT o.id,o.kind,o.instance_id,o.generation,o.history_expired_at,o.request_hash,${operationPayloadSql}=CAST(? AS TEXT) AS same_payload FROM ec_host_operations o WHERE o.host_id=? AND o.idempotency_key=?`).get(payload, hostId, options.idempotencyKey) as { id: string; kind: string; instance_id: string | null; generation: number | null; history_expired_at: string | null; request_hash: string | null; same_payload: number } | undefined; if (existing) { const samePayload = existing.request_hash ? existing.request_hash === createHash('sha256').update(payload).digest('hex') : !!existing.same_payload; if (existing.kind !== kind || existing.instance_id !== (options.instanceId ?? null) || existing.generation !== (options.generation ?? null) || !samePayload) throw new Error('Operation idempotency key is already associated with a different request.'); return getHostOperationStatus(existing.id)!; } } if (['instance.create','create'].includes(kind)) requireReadyHost(hostId); if (host.mode === 'disabled' && kind !== 'instance.cancel-create' && !(host.enablingSince && ['runtime.pull','library.download','plutonium.prepare','host.inspect'].includes(kind))) throw new Error('Host is disabled. Enable it before requesting operations.'); const requestHash = createHash('sha256').update(payload).digest('hex'); const packed = packFileJson(JSON.parse(payload.toString())); payload = JSON.stringify(packed.value); const id = options.id ?? randomUUID(); const timestamp = now(); const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?') .get(options.instanceId) as { server: string } | undefined)?.server ?? null : null; database().prepare(`INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at,source,server_name,request_hash) VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?,?,?,?)`) .run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, routingPayload ?? payload, timestamp, timestamp, (currentOperationSource() ?? 'Controller').slice(0, 200), serverName, requestHash); referenceBlobs('operation', id, packed.hashes); if (routingPayload !== undefined) database().prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))').run(id, payload); database().prepare(`UPDATE ec_host_operations AS o SET routine=CASE WHEN ${routineObservationSql} THEN 1 ELSE 0 END, target_path=${operationTargetSql} WHERE o.id=?`).run(id); database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp); return getHostOperationStatus(id)!; })(); queueMicrotask(() => workEvents.emit(hostId)); return operation; } export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'health' | 'write' = 'all'): HostOperation[] { return pendingHostOperationRows(hostId, lane).map(operationFromRow); } type OperationLane = 'all' | 'read' | 'health' | 'write'; export function pendingHostOperationRows(hostId: string, lane?: OperationLane, includePayload?: true): OperationRow[]; export function pendingHostOperationRows(hostId: string, lane: OperationLane, includePayload: false): OperationDeliveryRow[]; /** Delivery snapshots retain status/results while large immutable payloads stay in SQLite. */ export function pendingHostOperationRows(hostId: string, lane: OperationLane = 'all', includePayload = true): OperationDeliveryRow[] { const host = requireHost(hostId); if ((host.mode === 'disabled' && !host.enablingSince) || !host.connected) return []; // Older agents hold activity.lock across the entire delivered batch, including // observations. Filtering only the unsupported cleanup can keep the updater // busy indefinitely. Drain the old batch by withholding ALL further work; // heartbeats and receipts still flow, and the new capability releases this hold. // This must live in the controller so the fix also works before an agent update. if (database().prepare(`SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown') AND (NOT ? OR (json_extract(payload,'$.recoveryRebuildOperationId') IS NOT NULL AND NOT ?)) LIMIT 1`) .get(hostId, Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false), Number(host.inventory.capabilities?.includes('instance.unhealthy-recovery') ?? false))) return []; // Accepted/running/unknown commands keep the same immutable ID. The agent // consults its durable journal before acknowledging or executing a replay. const healthOnly = `(kind IN ('instance.inspect','rcon.status-batch') OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`; const healthLane = host.inventory.capabilities?.includes('operations.health-lane'); const laneFilter = lane === 'all' ? '' : lane === 'health' ? ` AND ${healthOnly}` : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnlyOperationSql}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`; // Health never queues behind a delivered file batch. Keep background reads // in small batches to amortize delivery without handing out a long backlog. const limit = healthLane ? lane === 'health' ? 1 : lane === 'read' ? 4 : 25 : 25; return database().transaction(() => { const columns = includePayload ? `${operationStatusColumns},${operationPayloadSql} AS payload` : operationStatusColumns; const rows = database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown') ${laneFilter} ${host.mode === 'disabled' ? "AND kind IN ('runtime.pull','library.download','plutonium.prepare','host.inspect')" : ''} AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create')) AND (kind!='instance.cancel-create' OR ?) AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id) AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cancellation JOIN native_provisioning creation ON creation.deployment_id=cancellation.creation_id WHERE creation.operation_id=o.id) ${lane === 'health' ? "AND (state NOT IN ('accepted','running') OR MAX(updated_at,COALESCE(last_delivered_at,'')) <= ?)" : ''} ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 WHEN kind='plutonium.prepare' THEN 2 ELSE 1 END, COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`) // A short replay lease avoids handing the idle health lane its busy // sibling's work, but still recovers a lost receipt or agent restart. .all(hostId, Number(host.readiness === 'ready' || host.mode === 'draining'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false), ...(lane === 'health' ? [new Date(Date.now() - 30_000).toISOString()] : [])) as OperationDeliveryRow[]; const preparationLimit = fleetSettings().preparationConcurrency; const libraryKinds = ['library.download','library.import']; const preparing = preparationLimit && rows.some(operation => libraryKinds.includes(operation.kind)) ? new Set((database().prepare(`SELECT DISTINCT host_id FROM ec_host_operations WHERE kind IN ('library.download','library.import') AND last_delivered_at IS NOT NULL AND state IN ('pending','accepted','running','unknown')`).all() as { host_id: string }[]).map(row => row.host_id)) : new Set(); const admitted = rows.filter(operation => { if (!libraryKinds.includes(operation.kind) || preparing.has(hostId) || !preparationLimit) return true; if (preparing.size >= preparationLimit) return false; preparing.add(hostId); return true; }); const deliveredAt = now(); for (const operation of admitted) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id); return admitted; })(); } /** Payloads never change after enqueueing; fetch one delivery's UTF-8 bytes without decoding. */ export function getHostOperationPayload(hostId: string, id: string): Buffer { const row = database().prepare(`SELECT CAST(${operationPayloadSql} AS BLOB) AS payload FROM ec_host_operations o WHERE o.id=? AND o.host_id=?`) .get(id, hostId) as { payload: Buffer } | undefined; if (!row) throw new Error('Host operation was not found for this host.'); if (!row.payload.includes('"contentSha256"') || getHost(hostId)?.inventory.capabilities?.includes('files.blobs')) return row.payload; return Buffer.from(JSON.stringify(unpackFileJson(JSON.parse(row.payload.toString())))); } export function reportHostOperationStatus(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperationStatus { const operation = database().transaction(() => { requireHost(hostId); const operation = getHostOperationStatus(id); if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.'); if (operation.kind === 'host.decommission' && input.state === 'completed') throw new Error('Host removal requires the cleanup worker completion receipt.'); if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.'); if (['completed','failed'].includes(operation.state)) return operation; const state = input.state as HostOperationState; if (operation.state === 'running' && state === 'accepted') return operation; const error = input.error === undefined || input.error === null ? null : String(input.error).slice(0, 4000); let result = input.result === undefined ? null : JSON.stringify(input.result); if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.'); if (result !== null) { const packed = packFileJson(input.result); result = JSON.stringify(packed.value); referenceBlobs('operation-result', id, packed.hashes); } const observation = state === 'completed' && !error && successfulObservation(operation.kind, operation.kind === 'rcon.execute' ? JSON.parse(getHostOperationPayload(hostId, id).toString('utf8')) : {}, input.result); database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=?,observation_success=? WHERE id=?') .run(state, result, error, now(), Number(observation), id); if (operation.kind === 'host.decommission') { if (state === 'failed') database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?").run(error ?? 'The agent could not start cleanup. Repair the installation, then retry deletion.', now(), id); else database().prepare("UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')").run(now(), id); } if (operation.state !== state) { database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)').run(id, state, now(), error); if (operation.kind.startsWith('instance.') && operation.kind !== 'instance.inspect' || operation.kind === 'files.stage') { recordAgentReceipt(id, state, now()); } } return getHostOperationStatus(id)!; })(); queueMicrotask(() => workEvents.emit(`operation:${id}`)); return operation; } export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation { reportHostOperationStatus(hostId, id, input); return getHostOperation(id)!; } export async function waitHostOperationStatus(id: string, timeoutMs = 30_000): Promise { const deadline = Date.now() + timeoutMs; while (true) { const operation = getHostOperationStatus(id); if (!operation) throw new Error('Host operation was not found.'); if (['completed','failed','unknown'].includes(operation.state)) return operation; if (Date.now() >= deadline) { // Delivery and expiry both claim the pending row transactionally. // Never cancel a delivered check or a command with side effects. const expired = database().transaction(() => { const timestamp = now(); const error = 'Expired before delivery.'; const result = database().prepare(`UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=? AND state='pending' AND last_delivered_at IS NULL AND kind IN ('instance.inspect','rcon.status-batch')`).run(error, timestamp, id); if (result.changes) database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)') .run(id, 'failed', timestamp, error); return result.changes > 0; })(); if (expired) { queueMicrotask(() => workEvents.emit(`operation:${id}`)); throw new Error(`Host operation ${id} expired before delivery.`); } throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`); } await waitForHostWork(`operation:${id}`, new AbortController().signal, Math.min(1000, Math.max(1, deadline - Date.now()))); } } export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise { await waitHostOperationStatus(id, timeoutMs); return getHostOperation(id)!; } export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean { const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress; if (host.decommission) return false; if (candidate < host.portMin || candidate > host.portMax) return false; const rows = database().prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND protocol=? AND port=?').all(hostId, protocol, candidate) as { instance_id: string; bind_address: string }[]; return !rows.some(row => row.instance_id !== instanceId && (row.bind_address === bind || ['0.0.0.0','::'].includes(bind) || ['0.0.0.0','::'].includes(row.bind_address))); } export function nextHostPort(hostId: string, additionalPorts: number[] = [], bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp'): number | null { const host = requireHost(hostId); for (let candidate = host.portMin; candidate <= host.portMax; candidate++) if (!additionalPorts.includes(candidate) && hostPortAvailable(hostId, candidate, bindAddress, protocol)) return candidate; return null; } export function reserveHostPort(hostId: string, instanceId: string, options: { port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): number { const host = requireHost(hostId); const bind = options.bindAddress ?? host.bindAddress; const protocol = options.protocol ?? 'udp'; if (host.decommission) throw new Error('Host cleanup is in progress; new port reservations are blocked.'); if (!isIP(bind)) throw new Error('Bind address must be an IP address.'); if (!['udp','tcp'].includes(protocol)) throw new Error('Port protocol must be UDP or TCP.'); if (!instanceId) throw new Error('An immutable instance ID is required.'); return database().transaction(() => { const existing = database().prepare('SELECT port FROM ec_host_ports WHERE host_id=? AND instance_id=? AND bind_address=? AND protocol=?').get(hostId, instanceId, bind, protocol) as { port: number } | undefined; if (existing && (options.port === undefined || options.port === existing.port)) return existing.port; const allocated = options.port ?? nextHostPort(hostId, [], bind, protocol); if (allocated === null || !hostPortAvailable(hostId, allocated, bind, protocol, instanceId)) throw new Error('No available game port on the selected host.'); database().prepare('INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) VALUES(?,?,?,?,?,?)').run(hostId, instanceId, bind, protocol, allocated, now()); scheduleAvailabilityRefresh(); return allocated; })(); } export function releaseHostPorts(instanceId: string): void { database().prepare('DELETE FROM ec_host_ports WHERE instance_id=?').run(instanceId); scheduleAvailabilityRefresh(); }