Automatically place servers using learned resource capacity

src/lib/server/hosts/capacity.ts

@@ -0,0 +1,83 @@
+import { db } from '../db/index.js';
+import { listHosts, requireReadyHost } from './index.js';
+import { requireReadyLibrary } from '../game-files/index.js';
+import type { Host } from '../../hosts.js';
+import { capacityCalibration } from './calibration.js';
+
+// Storage and OS headroom are separate from learned instance CPU/memory usage.
+const INSTANCE_STORAGE_BYTES = 1024 ** 3;
+const MEMORY_HEADROOM = 512 * 1024 ** 2;
+export class CapacityError extends Error {}
+export interface PlacementOptions {
+    hostId?: string; libraryId: string; image: string; bindAddress?: string;
+    thorough?: boolean; excludeInstanceId?: string;
+    planned?: { hostId: string; port: number }[];
+}
+
+/** One indexed port read per host, including both protocols and wildcard binds. */
+export function availableHostPort(host: Host, options: Pick<PlacementOptions, 'bindAddress' | 'excludeInstanceId' | 'planned'> = {}): number | null {
+    const bind = options.bindAddress ?? host.bindAddress;
+    const rows = db().prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?')
+        .all(host.id, options.excludeInstanceId ?? '') as { port: number; bind_address: string }[];
+    const used = new Set(rows.filter(row => row.bind_address === bind || ['0.0.0.0', '::'].includes(bind)
+        || ['0.0.0.0', '::'].includes(row.bind_address)).map(row => row.port));
+    for (const item of options.planned ?? []) if (item.hostId === host.id) used.add(item.port);
+    for (let port = host.portMin; port <= host.portMax; port++) if (!used.has(port)) return port;
+    return null;
+}
+
+function projectedLoad(host: Host, options: PlacementOptions): number {
+    const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
+    if (!cpuCount || cpuUsagePercent === undefined || !memoryTotal || memoryAvailable === undefined || memoryAvailable > memoryTotal) {
+        throw new Error('Waiting for valid CPU and memory measurements.');
+    }
+    let cpu = cpuUsagePercent;
+    let available = memoryAvailable;
+    if (options.thorough) {
+        const history = db().prepare(`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory
+            FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`)
+            .get(host.id, new Date(Date.now() - 5 * 60_000).toISOString(), cpuCount, memoryTotal) as { cpu: number | null; memory: number | null };
+        cpu = Math.max(cpu, history.cpu ?? cpu);
+        available = Math.min(available, history.memory ?? available);
+    }
+    // Unfinished/uncertain launches remain charged until recovery and a later
+    // heartbeat. Observed usage accounts for established servers; the count
+    // floor also reserves their minimum footprint while they are idle/stopped.
+    const instances = db().prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(NOT EXISTS (
+        SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
+        WHERE n.instance_id=i.id AND d.action='create' AND d.state IN ('done','warning') AND d.finished_at<?
+    )),0) AS pending FROM native_instances i WHERE i.host_id=? AND i.retired_at IS NULL AND i.id!=?`)
+        .get(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as { total: number; pending: number };
+    const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
+    const count = instances.total + planned + 1;
+    const pending = instances.pending + planned + 1;
+    const { running, startup } = capacityCalibration(host.id);
+    const cores = Math.max(cpuCount * cpu / 100 + pending * startup.cpuCores,
+        (count - pending) * running.cpuCores + pending * startup.cpuCores);
+    const memory = Math.max(memoryTotal - available + pending * startup.memoryBytes,
+        (count - pending) * running.memoryBytes + pending * startup.memoryBytes);
+    if (cores > cpuCount * 0.9) throw new Error('Insufficient CPU capacity.');
+    if (memory > memoryTotal - MEMORY_HEADROOM) throw new Error('Insufficient memory capacity.');
+    return Math.max(cores / cpuCount, memory / memoryTotal);
+}
+
+/** Admission stops at the first fit; placement evaluates and ranks every fit. No remote I/O. */
+export function choosePlacement(options: PlacementOptions): { host: Host; port: number; load: number } {
+    const candidates: { host: Host; port: number; load: number }[] = [];
+    const reasons: string[] = [];
+    for (const host of listHosts().filter(host => !options.hostId || host.id === options.hostId)) {
+        try {
+            requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES });
+            requireReadyLibrary(host.id, options.libraryId);
+            const port = availableHostPort(host, options);
+            if (port === null) throw new Error('No available UDP/TCP game port.');
+            const load = projectedLoad(host, options);
+            const candidate = { host, port, load };
+            if (!options.thorough) return candidate;
+            candidates.push(candidate);
+        } catch (error) { reasons.push(`${host.name}: ${(error as Error).message}`); }
+    }
+    candidates.sort((a, b) => a.load - b.load || a.host.id.localeCompare(b.host.id));
+    if (candidates.length) return candidates[0];
+    throw new CapacityError(`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`);
+}