import { db } from '../db/index.js'; import { listHosts, requireReadyHost } from './index.js'; import { assertReadyLibrary } from '../game-files/index.js'; import type { Host } from '../../hosts.js'; import { capacityCalibration } from './calibration.js'; // Storage and OS headroom are separate from learned instance CPU/memory usage. const INSTANCE_STORAGE_BYTES = 1024 ** 3; const MEMORY_HEADROOM = 500_000_000; const CPU_USABLE_FRACTION = 0.95; export class CapacityError extends Error {} export interface PlacementOptions { hostId?: string; libraryId: string; image: string; bindAddress?: string; excludeHostIds?: string[]; thorough?: boolean; excludeInstanceId?: string; // Execution only: not-yet-submitted launches wait their turn instead of // requiring their entire backlog to fit before any one can start. dispatch?: boolean; planned?: { hostId: string; port: number }[]; } /** One indexed port read per host, including both protocols and wildcard binds. */ export function availableHostPort(host: Host, options: Pick = {}): number | null { const bind = options.bindAddress ?? host.bindAddress; const rows = db().prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?') .all(host.id, options.excludeInstanceId ?? '') as { port: number; bind_address: string }[]; const used = new Set(rows.filter(row => row.bind_address === bind || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(row.bind_address)).map(row => row.port)); for (const item of options.planned ?? []) if (item.hostId === host.id) used.add(item.port); for (let port = host.portMin; port <= host.portMax; port++) if (!used.has(port)) return port; return null; } function projectedLoad(host: Host, options: PlacementOptions): number { const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory; if (!cpuCount || cpuUsagePercent === undefined || !memoryTotal || memoryAvailable === undefined || memoryAvailable > memoryTotal) { throw new Error('Waiting for valid CPU and memory measurements.'); } let historicalCpu = cpuUsagePercent; let historicalAvailable = memoryAvailable; if (options.thorough) { const history = db().prepare(`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`) .get(host.id, new Date(Date.now() - 5 * 60_000).toISOString(), cpuCount, memoryTotal) as { cpu: number | null; memory: number | null }; historicalCpu = Math.max(cpuUsagePercent, history.cpu ?? cpuUsagePercent); historicalAvailable = Math.min(memoryAvailable, history.memory ?? memoryAvailable); } const instances = db().prepare(`SELECT i.id, i.generation, NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id WHERE n.instance_id=i.id AND n.generation=i.generation AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at item.hostId === host.id).length ?? 0; // Admission budgets the eventual running collection. Dispatch budgets only // the next startup; the rest of the queue waits for live capacity to recover. const launch = options.dispatch ? startup : running; let extraCpu = planned * running.cpuCores + launch.cpuCores + incoming.n * startup.cpuCores; let extraMemory = planned * running.memoryBytes + launch.memoryBytes + incoming.n * startup.memoryBytes; let reservations = planned + 1 + incoming.n; const measurements = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample])); for (const instance of instances) { if (options.dispatch && options.excludeInstanceId && instance.unsubmitted) continue; if (instance.unsubmitted) { extraCpu += running.cpuCores; extraMemory += running.memoryBytes; reservations++; continue; } const sample = measurements.get(instance.id); // Current host usage already includes observed containers, including // those still awaiting health verification. Do not top up to old peaks. if (sample?.generation === instance.generation || !instance.pending) continue; // A submitted launch absent from current telemetry may still start. extraCpu += startup.cpuCores; extraMemory += startup.memoryBytes; reservations++; } const cores = cpuCount * cpuUsagePercent / 100 + extraCpu; const usedMemory = memoryTotal - memoryAvailable; const memory = usedMemory + extraMemory; const usableMemory = memoryTotal - MEMORY_HEADROOM; const gib = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)} GiB`; if (cores > cpuCount * CPU_USABLE_FRACTION) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.`); if (memory > usableMemory) throw new Error(`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. ` + `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(launch.memoryBytes)} ${options.dispatch ? 'startup' : 'running'} allowance).`); // Recovered resources must not remain unavailable until the historical // average catches up. History ranks hosts after current capacity passes. const historicalMemory = memoryTotal - historicalAvailable + extraMemory; const historicalCores = cpuCount * historicalCpu / 100 + extraCpu; return Math.max(historicalCores / cpuCount, historicalMemory / memoryTotal); } /** Admission stops at the first fit; placement evaluates and ranks every fit. No remote I/O. */ export function choosePlacement(options: PlacementOptions): { host: Host; port: number; load: number } { const candidates: { host: Host; port: number; load: number }[] = []; const reasons: string[] = []; for (const host of listHosts().filter(host => (!options.hostId || host.id === options.hostId) && !options.excludeHostIds?.includes(host.id))) { try { requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES }); assertReadyLibrary(host.id, options.libraryId); const port = availableHostPort(host, options); if (port === null) throw new Error('No available UDP/TCP game port.'); const load = projectedLoad(host, options); const candidate = { host, port, load }; if (!options.thorough) return candidate; candidates.push(candidate); } catch (error) { reasons.push(`${host.name}: ${(error as Error).message}`); } } candidates.sort((a, b) => a.load - b.load || a.host.id.localeCompare(b.host.id)); if (candidates.length) return candidates[0]; throw new CapacityError(`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`); }