Automatically place servers using learned resource capacity

src/lib/server/hosts/calibration.ts

@@ -0,0 +1,106 @@
+import { db } from '../db/index.js';
+import { ensureHostSchema } from './schema.js';
+import type { CapacityCalibration, HostInventory, ResourceAllowance } from '../../hosts.js';
+
+const BOOTSTRAP_CPU = 0.5;
+const BOOTSTRAP_MEMORY = 1024 ** 3;
+const MARGIN = 1.25;
+const WINDOW_MS = 24 * 60 * 60_000;
+const MAX_SAMPLES = 10_000;
+type Phase = 'startup' | 'running';
+interface AllowanceRow {
+    scope: string; phase: Phase; cpu_cores: number; memory_bytes: number; samples: number;
+    window_start: string; observed_at: string; computed_at: string;
+}
+function database() { const handle = db(); ensureHostSchema(handle); return handle; }
+
+export function validateInstanceResources(inventory: HostInventory): void {
+    const samples = inventory.instanceResources;
+    if (samples === undefined) return;
+    if (!Array.isArray(samples) || samples.length > 1000) throw new Error('Invalid instance resource samples.');
+    const seen = new Set<string>();
+    for (const sample of samples) {
+        if (!sample || typeof sample !== 'object' || typeof sample.instanceId !== 'string' || !sample.instanceId || sample.instanceId.length > 100
+            || !Number.isSafeInteger(sample.generation) || sample.generation < 1
+            || !Number.isSafeInteger(sample.uptimeSeconds) || sample.uptimeSeconds < 0
+            || typeof sample.cpuCores !== 'number' || !Number.isFinite(sample.cpuCores) || sample.cpuCores < 0
+            || !Number.isSafeInteger(sample.memoryBytes) || sample.memoryBytes <= 0
+            || (inventory.cpuCount !== undefined && sample.cpuCores > inventory.cpuCount + 0.01)
+            || (inventory.memoryTotal !== undefined && sample.memoryBytes > inventory.memoryTotal)
+            || seen.has(sample.instanceId)) throw new Error('Invalid instance resource sample.');
+        seen.add(sample.instanceId);
+    }
+}
+
+/** Heartbeat ingestion owns all learning work. Launch requests only read cached allowances. */
+export function recordInstanceResources(hostId: string, inventory: HostInventory, timestamp: string): void {
+    if (!inventory.instanceResources?.length || !inventory.dockerRunning || !inventory.cpuCount || !inventory.memoryTotal) return;
+    const handle = database();
+    const bucket = timestamp.slice(0, 16) + ':00.000Z';
+    const observed = new Map<Phase, { cpu: number; memory: number }>();
+    const owned = handle.prepare('SELECT 1 FROM native_instances WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL');
+    const insert = handle.prepare(`INSERT INTO ec_instance_resource_samples
+        (host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,?,?,?,?,?,?)
+        ON CONFLICT(host_id,instance_id,generation,phase,bucket) DO UPDATE SET
+            cpu_cores=MAX(cpu_cores,excluded.cpu_cores), memory_bytes=MAX(memory_bytes,excluded.memory_bytes)`);
+    for (const sample of inventory.instanceResources) {
+        // An agent can only train from its currently owned generation; foreign,
+        // retired and delayed pre-rebuild samples cannot contaminate the model.
+        if (!owned.get(sample.instanceId, hostId, sample.generation)) continue;
+        const phase: Phase = sample.uptimeSeconds < 300 ? 'startup' : 'running';
+        insert.run(hostId, sample.instanceId, sample.generation, phase, bucket, sample.cpuCores, sample.memoryBytes);
+        const prior = observed.get(phase);
+        observed.set(phase, { cpu: Math.max(prior?.cpu ?? 0, sample.cpuCores), memory: Math.max(prior?.memory ?? 0, sample.memoryBytes) });
+    }
+    for (const [phase, peak] of observed) for (const scope of [hostId, '*']) {
+        const previous = handle.prepare('SELECT * FROM ec_resource_allowances WHERE scope=? AND phase=?').get(scope, phase) as AllowanceRow | undefined;
+        if (!previous || previous.computed_at < bucket) {
+            const rows = handle.prepare(`SELECT cpu_cores, memory_bytes, bucket FROM ec_instance_resource_samples
+                WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`)
+                .all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as
+                { cpu_cores: number; memory_bytes: number; bucket: string }[];
+            const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1];
+            const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu) * MARGIN;
+            const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory) * MARGIN);
+            handle.prepare(`INSERT INTO ec_resource_allowances
+                (scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(scope,phase) DO UPDATE SET
+                cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples,
+                window_start=excluded.window_start,observed_at=excluded.observed_at,computed_at=excluded.computed_at`)
+                .run(scope, phase, cpu, memory, rows.length, rows.at(-1)!.bucket, timestamp, timestamp);
+            if (scope === '*') handle.prepare('DELETE FROM ec_instance_resource_samples WHERE bucket<?')
+                .run(new Date(Date.parse(timestamp) - WINDOW_MS).toISOString());
+        } else {
+            // Raise the allowance immediately on a new peak; a lower sample
+            // cannot reduce it until the next scheduled percentile calculation.
+            handle.prepare(`UPDATE ec_resource_allowances SET cpu_cores=MAX(cpu_cores,?),memory_bytes=MAX(memory_bytes,?),observed_at=? WHERE scope=? AND phase=?`)
+                .run(peak.cpu * MARGIN, Math.ceil(peak.memory * MARGIN), timestamp, scope, phase);
+        }
+    }
+}
+
+/** Host measurements account for hardware differences; fleet evidence covers new hosts. */
+export function capacityCalibration(hostId: string): CapacityCalibration {
+    const rows = database().prepare('SELECT * FROM ec_resource_allowances WHERE scope IN (?,?) AND observed_at>=?')
+        .all(hostId, '*', new Date(Date.now() - WINDOW_MS).toISOString()) as AllowanceRow[];
+    const allowance = (phase: Phase): ResourceAllowance => {
+        const local = rows.find(row => row.scope === hostId && row.phase === phase);
+        const fleet = rows.find(row => row.scope === '*' && row.phase === phase);
+        const learned = (row?: AllowanceRow) => !!row && row.samples >= (phase === 'startup' ? 3 : 20)
+            && Date.parse(row.observed_at) - Date.parse(row.window_start) >= (phase === 'startup' ? 120_000 : 300_000);
+        const selected = learned(local) ? local : learned(fleet) ? fleet : undefined;
+        return {
+            // A few higher observations can increase a fleet/bootstrap allowance
+            // immediately. Decreasing it requires the evidence thresholds above.
+            cpuCores: Math.max(0.01, selected?.cpu_cores ?? BOOTSTRAP_CPU, local?.cpu_cores ?? 0, selected ? 0 : fleet?.cpu_cores ?? 0),
+            memoryBytes: Math.max(selected?.memory_bytes ?? BOOTSTRAP_MEMORY, local?.memory_bytes ?? 0, selected ? 0 : fleet?.memory_bytes ?? 0),
+            source: selected ? selected.scope === hostId ? 'host' : 'fleet' : 'bootstrap',
+            samples: selected?.samples ?? local?.samples ?? fleet?.samples ?? 0,
+            observedAt: selected?.observed_at ?? local?.observed_at ?? fleet?.observed_at ?? null
+        };
+    };
+    const running = allowance('running'); const startup = allowance('startup');
+    if (running.source === 'bootstrap' && (running.cpuCores > startup.cpuCores || running.memoryBytes > startup.memoryBytes)) startup.source = 'bootstrap';
+    startup.cpuCores = Math.max(startup.cpuCores, running.cpuCores);
+    startup.memoryBytes = Math.max(startup.memoryBytes, running.memoryBytes);
+    return { running, startup };
+}

src/lib/server/hosts/capacity.ts

@@ -0,0 +1,83 @@
+import { db } from '../db/index.js';
+import { listHosts, requireReadyHost } from './index.js';
+import { requireReadyLibrary } from '../game-files/index.js';
+import type { Host } from '../../hosts.js';
+import { capacityCalibration } from './calibration.js';
+
+// Storage and OS headroom are separate from learned instance CPU/memory usage.
+const INSTANCE_STORAGE_BYTES = 1024 ** 3;
+const MEMORY_HEADROOM = 512 * 1024 ** 2;
+export class CapacityError extends Error {}
+export interface PlacementOptions {
+    hostId?: string; libraryId: string; image: string; bindAddress?: string;
+    thorough?: boolean; excludeInstanceId?: string;
+    planned?: { hostId: string; port: number }[];
+}
+
+/** One indexed port read per host, including both protocols and wildcard binds. */
+export function availableHostPort(host: Host, options: Pick<PlacementOptions, 'bindAddress' | 'excludeInstanceId' | 'planned'> = {}): number | null {
+    const bind = options.bindAddress ?? host.bindAddress;
+    const rows = db().prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?')
+        .all(host.id, options.excludeInstanceId ?? '') as { port: number; bind_address: string }[];
+    const used = new Set(rows.filter(row => row.bind_address === bind || ['0.0.0.0', '::'].includes(bind)
+        || ['0.0.0.0', '::'].includes(row.bind_address)).map(row => row.port));
+    for (const item of options.planned ?? []) if (item.hostId === host.id) used.add(item.port);
+    for (let port = host.portMin; port <= host.portMax; port++) if (!used.has(port)) return port;
+    return null;
+}
+
+function projectedLoad(host: Host, options: PlacementOptions): number {
+    const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
+    if (!cpuCount || cpuUsagePercent === undefined || !memoryTotal || memoryAvailable === undefined || memoryAvailable > memoryTotal) {
+        throw new Error('Waiting for valid CPU and memory measurements.');
+    }
+    let cpu = cpuUsagePercent;
+    let available = memoryAvailable;
+    if (options.thorough) {
+        const history = db().prepare(`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory
+            FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`)
+            .get(host.id, new Date(Date.now() - 5 * 60_000).toISOString(), cpuCount, memoryTotal) as { cpu: number | null; memory: number | null };
+        cpu = Math.max(cpu, history.cpu ?? cpu);
+        available = Math.min(available, history.memory ?? available);
+    }
+    // Unfinished/uncertain launches remain charged until recovery and a later
+    // heartbeat. Observed usage accounts for established servers; the count
+    // floor also reserves their minimum footprint while they are idle/stopped.
+    const instances = db().prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(NOT EXISTS (
+        SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
+        WHERE n.instance_id=i.id AND d.action='create' AND d.state IN ('done','warning') AND d.finished_at<?
+    )),0) AS pending FROM native_instances i WHERE i.host_id=? AND i.retired_at IS NULL AND i.id!=?`)
+        .get(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as { total: number; pending: number };
+    const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
+    const count = instances.total + planned + 1;
+    const pending = instances.pending + planned + 1;
+    const { running, startup } = capacityCalibration(host.id);
+    const cores = Math.max(cpuCount * cpu / 100 + pending * startup.cpuCores,
+        (count - pending) * running.cpuCores + pending * startup.cpuCores);
+    const memory = Math.max(memoryTotal - available + pending * startup.memoryBytes,
+        (count - pending) * running.memoryBytes + pending * startup.memoryBytes);
+    if (cores > cpuCount * 0.9) throw new Error('Insufficient CPU capacity.');
+    if (memory > memoryTotal - MEMORY_HEADROOM) throw new Error('Insufficient memory capacity.');
+    return Math.max(cores / cpuCount, memory / memoryTotal);
+}
+
+/** Admission stops at the first fit; placement evaluates and ranks every fit. No remote I/O. */
+export function choosePlacement(options: PlacementOptions): { host: Host; port: number; load: number } {
+    const candidates: { host: Host; port: number; load: number }[] = [];
+    const reasons: string[] = [];
+    for (const host of listHosts().filter(host => !options.hostId || host.id === options.hostId)) {
+        try {
+            requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES });
+            requireReadyLibrary(host.id, options.libraryId);
+            const port = availableHostPort(host, options);
+            if (port === null) throw new Error('No available UDP/TCP game port.');
+            const load = projectedLoad(host, options);
+            const candidate = { host, port, load };
+            if (!options.thorough) return candidate;
+            candidates.push(candidate);
+        } catch (error) { reasons.push(`${host.name}: ${(error as Error).message}`); }
+    }
+    candidates.sort((a, b) => a.load - b.load || a.host.id.localeCompare(b.host.id));
+    if (candidates.length) return candidates[0];
+    throw new CapacityError(`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`);
+}

src/lib/server/hosts/index.ts

@@ -4,6 +4,7 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypt
 import { isIP } from 'node:net';
 import { db } from '../db/index.js';
 import { ensureHostSchema } from './schema.js';
+import { recordInstanceResources, validateInstanceResources } from './calibration.js';
 import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
 
 export const AGENT_PROTOCOL_VERSION = 1;
@@ -219,6 +220,7 @@ export function validateInventory(value: unknown): HostInventory {
         if (inventory[key] !== undefined && (typeof inventory[key] !== 'number' || !Number.isSafeInteger(inventory[key]) || Number(inventory[key]) < 0)) throw new Error(`Invalid inventory ${key}.`);
     }
     if (inventory.cpuUsagePercent !== undefined && (typeof inventory.cpuUsagePercent !== 'number' || !Number.isFinite(inventory.cpuUsagePercent) || inventory.cpuUsagePercent < 0 || inventory.cpuUsagePercent > 100)) throw new Error('Invalid inventory cpuUsagePercent.');
+    validateInstanceResources(inventory);
     for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
     if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
     if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
@@ -253,6 +255,7 @@ export function enrollHost(token: unknown, rawInventory: unknown): { hostId: str
         revokeCredentials(enrollment.host_id);
         const credential = issueCredential(enrollment.host_id);
         database().prepare('UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), now(), inventory.hostname ?? '', JSON.stringify(inventory), enrollment.host_id);
+        recordResources(enrollment.host_id, inventory, now());
         return { hostId: enrollment.host_id, credential, heartbeatSeconds: 15 };
     })();
 }
@@ -284,9 +287,23 @@ export function rotateCredential(hostId: string, nextCredential?: string): strin
 }
 export function heartbeat(hostId: string, value: unknown): Host {
     requireHost(hostId); const inventory = validateInventory(value);
-    database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), inventory.hostname ?? '', JSON.stringify(inventory), hostId);
+    database().transaction(() => {
+        const timestamp = now();
+        database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(timestamp, inventory.hostname ?? '', JSON.stringify(inventory), hostId);
+        recordResources(hostId, inventory, timestamp);
+    })();
     return requireHost(hostId);
 }
+
+function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void {
+    recordInstanceResources(hostId, inventory, timestamp);
+    database().prepare(`INSERT OR REPLACE INTO ec_host_resource_samples
+        (host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,?,?,?,?)`)
+        .run(hostId, timestamp, inventory.cpuCount ?? null, inventory.cpuUsagePercent ?? null,
+            inventory.memoryTotal ?? null, inventory.memoryAvailable ?? null);
+    database().prepare('DELETE FROM ec_host_resource_samples WHERE host_id=? AND sampled_at<?')
+        .run(hostId, new Date(Date.parse(timestamp) - 7 * 86400_000).toISOString());
+}
 export function requireReadyHost(hostId: string, options: { libraryId?: string; requiredBytes?: number; image?: string; port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): Host {
     const host = requireHost(hostId); const reasons = [...host.readinessReasons];
     if (options.requiredBytes !== undefined && (host.inventory.storageAvailable ?? 0) < options.requiredBytes + host.minimumFreeBytes) reasons.push('Insufficient storage for this deployment.');

src/lib/server/hosts/schema.ts

@@ -40,6 +40,25 @@ export function ensureHostSchema(handle: Database.Database): void {
             PRIMARY KEY(host_id,bind_address,protocol,port)
         );
         CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id);
+        CREATE TABLE IF NOT EXISTS ec_host_resource_samples (
+            host_id TEXT NOT NULL REFERENCES ec_hosts(id), sampled_at TEXT NOT NULL,
+            cpu_count REAL, cpu_usage_percent REAL, memory_total REAL, memory_available REAL,
+            PRIMARY KEY(host_id, sampled_at)
+        );
+        CREATE TABLE IF NOT EXISTS ec_instance_resource_samples (
+            host_id TEXT NOT NULL, instance_id TEXT NOT NULL, generation INTEGER NOT NULL,
+            phase TEXT NOT NULL CHECK(phase IN ('startup','running')), bucket TEXT NOT NULL,
+            cpu_cores REAL NOT NULL, memory_bytes REAL NOT NULL,
+            PRIMARY KEY(host_id,instance_id,generation,phase,bucket)
+        );
+        CREATE INDEX IF NOT EXISTS ec_instance_resources_host ON ec_instance_resource_samples(host_id,phase,bucket DESC);
+        CREATE INDEX IF NOT EXISTS ec_instance_resources_fleet ON ec_instance_resource_samples(phase,bucket DESC);
+        CREATE INDEX IF NOT EXISTS ec_instance_resources_retention ON ec_instance_resource_samples(bucket);
+        CREATE TABLE IF NOT EXISTS ec_resource_allowances (
+            scope TEXT NOT NULL, phase TEXT NOT NULL, cpu_cores REAL NOT NULL, memory_bytes REAL NOT NULL,
+            samples INTEGER NOT NULL, window_start TEXT NOT NULL, observed_at TEXT NOT NULL, computed_at TEXT NOT NULL,
+            PRIMARY KEY(scope,phase)
+        );
         CREATE TABLE IF NOT EXISTS ec_host_decommissions (
             host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id),
             operation_id TEXT NOT NULL UNIQUE REFERENCES ec_host_operations(id),