Automatically place servers using learned resource capacity

src/lib/server/backends/schema.ts

@@ -9,6 +9,7 @@ export function migrateNativeInstances(handle: Database.Database): void {
             port INTEGER NOT NULL, public_address TEXT, public_port INTEGER,
             created_at TEXT NOT NULL, retired_at TEXT
         );
+        CREATE INDEX IF NOT EXISTS native_instance_host ON native_instances(host_id) WHERE retired_at IS NULL;
         CREATE UNIQUE INDEX IF NOT EXISTS native_instance_live_name ON native_instances(server) WHERE retired_at IS NULL;
         CREATE TABLE IF NOT EXISTS native_provisioning (
             deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
@@ -16,6 +17,7 @@ export function migrateNativeInstances(handle: Database.Database): void {
             stage_operation_id TEXT, operation_id TEXT, files_json TEXT,
             applied_at TEXT, verified_at TEXT
         );
+        CREATE INDEX IF NOT EXISTS native_provisioning_instance ON native_provisioning(instance_id);
         CREATE TABLE IF NOT EXISTS native_creation_cancellations (
             creation_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
             cleanup_id TEXT NOT NULL UNIQUE REFERENCES server_deployments(id)

src/lib/server/fleet/collections.ts

@@ -12,6 +12,7 @@ import { serverObservation } from './status.js';
 import { activeBetaRemovalBlocker, getServer } from './servers.js';
 import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
 import { nativeSelection, nextNativePort } from '../provisioning/native.js';
+import { listHosts } from '../hosts/index.js';
 import { nativeInstance } from '../backends/native.js';
 
 function coverage(entries: BlueprintEntry[]) {
@@ -73,15 +74,24 @@ function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode) {
 function allocate(entries: BlueprintEntry[], releasingServers: string[] = []): CollectionPreview['servers'] {
     const names: string[] = [];
     const ports: number[] = [];
+    const planned: { hostId: string; port: number }[] = [];
     const counts = coverage(entries);
     return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
         const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
         const lobby = nextLobbyNumber(base, names);
         const name = lobby === null ? null : `${base}_${lobby}`;
-        const port = entry.recipe.native ? nextNativePort(entry.recipe.native, ports) : nextAvailablePort(ports, releasingServers);
+        let native = entry.recipe.native;
+        let port: number | null;
+        if (native) {
+            try {
+                native = nativeSelection(native as unknown as Record<string, unknown>, entry.recipe.game, { planned });
+                port = nextNativePort(native, planned.filter(item => item.hostId === native!.hostId).map(item => item.port));
+                if (port !== null) planned.push({ hostId: native.hostId, port });
+            } catch { port = null; }
+        } else port = nextAvailablePort(ports, releasingServers);
         if (name) names.push(name);
         if (port !== null) ports.push(port);
-        return { entryIndex, name, port, hasSavedKey: !!name && !!getServerKey(name) };
+        return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && !!getServerKey(name) };
     }));
 }
 
@@ -98,7 +108,7 @@ function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEn
         if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.');
     }
     if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
-    if (servers.some(server => server.port === null)) blockers.push(`There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
+    if (servers.some(server => server.port === null)) blockers.push(entries.some(entry => entry.recipe.native) ? 'There is not enough host CPU, memory or port capacity for the whole collection.' : `There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
     return blockers;
 }
 
@@ -117,7 +127,7 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill',
     if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.');
     return db().transaction((): CollectionPreview => {
         const { entries, blueprintName } = blueprintEntries(blueprintId);
-        if (selection?.hostId !== undefined || !config.komodo.address) {
+        if (selection?.hostId !== undefined || !config.komodo.address || listHosts().length > 0) {
             const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
             for (const entry of entries) {
                 entry.recipe.native = nativeSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
@@ -159,7 +169,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
         const servers = allocate(entries, releasingServers);
         const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
         if (blockers.length) throw new Error(blockers.join(' '));
-        if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || server.port !== reviewed[i]?.port)) {
+        if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || (!server.native?.automatic && server.port !== reviewed[i]?.port))) {
             throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.');
         }
         const expected = new Set(servers.map(server => server.name));
@@ -196,7 +206,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
             return { id, ports };
         });
         servers.forEach((server, index) => {
-            const recipe = entries[server.entryIndex].recipe;
+            const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) };
             const childPreviewId = randomUUID();
             db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
                 .run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name);

src/lib/server/fleet/launch-recipes.ts

@@ -9,7 +9,7 @@ import { getBaselineForScope, getVersion, listPromotedOverlaysForScope } from '.
 import { sha256Buffer } from './files.js';
 
 export interface LaunchRecipe {
-    native?: { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number };
+    native?: import('../../hosts.js').NativePlacement;
     format: 1;
     game: ZombieGame;
     mapSlug: string;

src/lib/server/hosts/calibration.ts

@@ -0,0 +1,106 @@
+import { db } from '../db/index.js';
+import { ensureHostSchema } from './schema.js';
+import type { CapacityCalibration, HostInventory, ResourceAllowance } from '../../hosts.js';
+
+const BOOTSTRAP_CPU = 0.5;
+const BOOTSTRAP_MEMORY = 1024 ** 3;
+const MARGIN = 1.25;
+const WINDOW_MS = 24 * 60 * 60_000;
+const MAX_SAMPLES = 10_000;
+type Phase = 'startup' | 'running';
+interface AllowanceRow {
+    scope: string; phase: Phase; cpu_cores: number; memory_bytes: number; samples: number;
+    window_start: string; observed_at: string; computed_at: string;
+}
+function database() { const handle = db(); ensureHostSchema(handle); return handle; }
+
+export function validateInstanceResources(inventory: HostInventory): void {
+    const samples = inventory.instanceResources;
+    if (samples === undefined) return;
+    if (!Array.isArray(samples) || samples.length > 1000) throw new Error('Invalid instance resource samples.');
+    const seen = new Set<string>();
+    for (const sample of samples) {
+        if (!sample || typeof sample !== 'object' || typeof sample.instanceId !== 'string' || !sample.instanceId || sample.instanceId.length > 100
+            || !Number.isSafeInteger(sample.generation) || sample.generation < 1
+            || !Number.isSafeInteger(sample.uptimeSeconds) || sample.uptimeSeconds < 0
+            || typeof sample.cpuCores !== 'number' || !Number.isFinite(sample.cpuCores) || sample.cpuCores < 0
+            || !Number.isSafeInteger(sample.memoryBytes) || sample.memoryBytes <= 0
+            || (inventory.cpuCount !== undefined && sample.cpuCores > inventory.cpuCount + 0.01)
+            || (inventory.memoryTotal !== undefined && sample.memoryBytes > inventory.memoryTotal)
+            || seen.has(sample.instanceId)) throw new Error('Invalid instance resource sample.');
+        seen.add(sample.instanceId);
+    }
+}
+
+/** Heartbeat ingestion owns all learning work. Launch requests only read cached allowances. */
+export function recordInstanceResources(hostId: string, inventory: HostInventory, timestamp: string): void {
+    if (!inventory.instanceResources?.length || !inventory.dockerRunning || !inventory.cpuCount || !inventory.memoryTotal) return;
+    const handle = database();
+    const bucket = timestamp.slice(0, 16) + ':00.000Z';
+    const observed = new Map<Phase, { cpu: number; memory: number }>();
+    const owned = handle.prepare('SELECT 1 FROM native_instances WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL');
+    const insert = handle.prepare(`INSERT INTO ec_instance_resource_samples
+        (host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,?,?,?,?,?,?)
+        ON CONFLICT(host_id,instance_id,generation,phase,bucket) DO UPDATE SET
+            cpu_cores=MAX(cpu_cores,excluded.cpu_cores), memory_bytes=MAX(memory_bytes,excluded.memory_bytes)`);
+    for (const sample of inventory.instanceResources) {
+        // An agent can only train from its currently owned generation; foreign,
+        // retired and delayed pre-rebuild samples cannot contaminate the model.
+        if (!owned.get(sample.instanceId, hostId, sample.generation)) continue;
+        const phase: Phase = sample.uptimeSeconds < 300 ? 'startup' : 'running';
+        insert.run(hostId, sample.instanceId, sample.generation, phase, bucket, sample.cpuCores, sample.memoryBytes);
+        const prior = observed.get(phase);
+        observed.set(phase, { cpu: Math.max(prior?.cpu ?? 0, sample.cpuCores), memory: Math.max(prior?.memory ?? 0, sample.memoryBytes) });
+    }
+    for (const [phase, peak] of observed) for (const scope of [hostId, '*']) {
+        const previous = handle.prepare('SELECT * FROM ec_resource_allowances WHERE scope=? AND phase=?').get(scope, phase) as AllowanceRow | undefined;
+        if (!previous || previous.computed_at < bucket) {
+            const rows = handle.prepare(`SELECT cpu_cores, memory_bytes, bucket FROM ec_instance_resource_samples
+                WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`)
+                .all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as
+                { cpu_cores: number; memory_bytes: number; bucket: string }[];
+            const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1];
+            const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu) * MARGIN;
+            const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory) * MARGIN);
+            handle.prepare(`INSERT INTO ec_resource_allowances
+                (scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(scope,phase) DO UPDATE SET
+                cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples,
+                window_start=excluded.window_start,observed_at=excluded.observed_at,computed_at=excluded.computed_at`)
+                .run(scope, phase, cpu, memory, rows.length, rows.at(-1)!.bucket, timestamp, timestamp);
+            if (scope === '*') handle.prepare('DELETE FROM ec_instance_resource_samples WHERE bucket<?')
+                .run(new Date(Date.parse(timestamp) - WINDOW_MS).toISOString());
+        } else {
+            // Raise the allowance immediately on a new peak; a lower sample
+            // cannot reduce it until the next scheduled percentile calculation.
+            handle.prepare(`UPDATE ec_resource_allowances SET cpu_cores=MAX(cpu_cores,?),memory_bytes=MAX(memory_bytes,?),observed_at=? WHERE scope=? AND phase=?`)
+                .run(peak.cpu * MARGIN, Math.ceil(peak.memory * MARGIN), timestamp, scope, phase);
+        }
+    }
+}
+
+/** Host measurements account for hardware differences; fleet evidence covers new hosts. */
+export function capacityCalibration(hostId: string): CapacityCalibration {
+    const rows = database().prepare('SELECT * FROM ec_resource_allowances WHERE scope IN (?,?) AND observed_at>=?')
+        .all(hostId, '*', new Date(Date.now() - WINDOW_MS).toISOString()) as AllowanceRow[];
+    const allowance = (phase: Phase): ResourceAllowance => {
+        const local = rows.find(row => row.scope === hostId && row.phase === phase);
+        const fleet = rows.find(row => row.scope === '*' && row.phase === phase);
+        const learned = (row?: AllowanceRow) => !!row && row.samples >= (phase === 'startup' ? 3 : 20)
+            && Date.parse(row.observed_at) - Date.parse(row.window_start) >= (phase === 'startup' ? 120_000 : 300_000);
+        const selected = learned(local) ? local : learned(fleet) ? fleet : undefined;
+        return {
+            // A few higher observations can increase a fleet/bootstrap allowance
+            // immediately. Decreasing it requires the evidence thresholds above.
+            cpuCores: Math.max(0.01, selected?.cpu_cores ?? BOOTSTRAP_CPU, local?.cpu_cores ?? 0, selected ? 0 : fleet?.cpu_cores ?? 0),
+            memoryBytes: Math.max(selected?.memory_bytes ?? BOOTSTRAP_MEMORY, local?.memory_bytes ?? 0, selected ? 0 : fleet?.memory_bytes ?? 0),
+            source: selected ? selected.scope === hostId ? 'host' : 'fleet' : 'bootstrap',
+            samples: selected?.samples ?? local?.samples ?? fleet?.samples ?? 0,
+            observedAt: selected?.observed_at ?? local?.observed_at ?? fleet?.observed_at ?? null
+        };
+    };
+    const running = allowance('running'); const startup = allowance('startup');
+    if (running.source === 'bootstrap' && (running.cpuCores > startup.cpuCores || running.memoryBytes > startup.memoryBytes)) startup.source = 'bootstrap';
+    startup.cpuCores = Math.max(startup.cpuCores, running.cpuCores);
+    startup.memoryBytes = Math.max(startup.memoryBytes, running.memoryBytes);
+    return { running, startup };
+}

src/lib/server/hosts/capacity.ts

@@ -0,0 +1,83 @@
+import { db } from '../db/index.js';
+import { listHosts, requireReadyHost } from './index.js';
+import { requireReadyLibrary } from '../game-files/index.js';
+import type { Host } from '../../hosts.js';
+import { capacityCalibration } from './calibration.js';
+
+// Storage and OS headroom are separate from learned instance CPU/memory usage.
+const INSTANCE_STORAGE_BYTES = 1024 ** 3;
+const MEMORY_HEADROOM = 512 * 1024 ** 2;
+export class CapacityError extends Error {}
+export interface PlacementOptions {
+    hostId?: string; libraryId: string; image: string; bindAddress?: string;
+    thorough?: boolean; excludeInstanceId?: string;
+    planned?: { hostId: string; port: number }[];
+}
+
+/** One indexed port read per host, including both protocols and wildcard binds. */
+export function availableHostPort(host: Host, options: Pick<PlacementOptions, 'bindAddress' | 'excludeInstanceId' | 'planned'> = {}): number | null {
+    const bind = options.bindAddress ?? host.bindAddress;
+    const rows = db().prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?')
+        .all(host.id, options.excludeInstanceId ?? '') as { port: number; bind_address: string }[];
+    const used = new Set(rows.filter(row => row.bind_address === bind || ['0.0.0.0', '::'].includes(bind)
+        || ['0.0.0.0', '::'].includes(row.bind_address)).map(row => row.port));
+    for (const item of options.planned ?? []) if (item.hostId === host.id) used.add(item.port);
+    for (let port = host.portMin; port <= host.portMax; port++) if (!used.has(port)) return port;
+    return null;
+}
+
+function projectedLoad(host: Host, options: PlacementOptions): number {
+    const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
+    if (!cpuCount || cpuUsagePercent === undefined || !memoryTotal || memoryAvailable === undefined || memoryAvailable > memoryTotal) {
+        throw new Error('Waiting for valid CPU and memory measurements.');
+    }
+    let cpu = cpuUsagePercent;
+    let available = memoryAvailable;
+    if (options.thorough) {
+        const history = db().prepare(`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory
+            FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`)
+            .get(host.id, new Date(Date.now() - 5 * 60_000).toISOString(), cpuCount, memoryTotal) as { cpu: number | null; memory: number | null };
+        cpu = Math.max(cpu, history.cpu ?? cpu);
+        available = Math.min(available, history.memory ?? available);
+    }
+    // Unfinished/uncertain launches remain charged until recovery and a later
+    // heartbeat. Observed usage accounts for established servers; the count
+    // floor also reserves their minimum footprint while they are idle/stopped.
+    const instances = db().prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(NOT EXISTS (
+        SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
+        WHERE n.instance_id=i.id AND d.action='create' AND d.state IN ('done','warning') AND d.finished_at<?
+    )),0) AS pending FROM native_instances i WHERE i.host_id=? AND i.retired_at IS NULL AND i.id!=?`)
+        .get(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as { total: number; pending: number };
+    const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
+    const count = instances.total + planned + 1;
+    const pending = instances.pending + planned + 1;
+    const { running, startup } = capacityCalibration(host.id);
+    const cores = Math.max(cpuCount * cpu / 100 + pending * startup.cpuCores,
+        (count - pending) * running.cpuCores + pending * startup.cpuCores);
+    const memory = Math.max(memoryTotal - available + pending * startup.memoryBytes,
+        (count - pending) * running.memoryBytes + pending * startup.memoryBytes);
+    if (cores > cpuCount * 0.9) throw new Error('Insufficient CPU capacity.');
+    if (memory > memoryTotal - MEMORY_HEADROOM) throw new Error('Insufficient memory capacity.');
+    return Math.max(cores / cpuCount, memory / memoryTotal);
+}
+
+/** Admission stops at the first fit; placement evaluates and ranks every fit. No remote I/O. */
+export function choosePlacement(options: PlacementOptions): { host: Host; port: number; load: number } {
+    const candidates: { host: Host; port: number; load: number }[] = [];
+    const reasons: string[] = [];
+    for (const host of listHosts().filter(host => !options.hostId || host.id === options.hostId)) {
+        try {
+            requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES });
+            requireReadyLibrary(host.id, options.libraryId);
+            const port = availableHostPort(host, options);
+            if (port === null) throw new Error('No available UDP/TCP game port.');
+            const load = projectedLoad(host, options);
+            const candidate = { host, port, load };
+            if (!options.thorough) return candidate;
+            candidates.push(candidate);
+        } catch (error) { reasons.push(`${host.name}: ${(error as Error).message}`); }
+    }
+    candidates.sort((a, b) => a.load - b.load || a.host.id.localeCompare(b.host.id));
+    if (candidates.length) return candidates[0];
+    throw new CapacityError(`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`);
+}

src/lib/server/hosts/index.ts

@@ -4,6 +4,7 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypt
 import { isIP } from 'node:net';
 import { db } from '../db/index.js';
 import { ensureHostSchema } from './schema.js';
+import { recordInstanceResources, validateInstanceResources } from './calibration.js';
 import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
 
 export const AGENT_PROTOCOL_VERSION = 1;
@@ -219,6 +220,7 @@ export function validateInventory(value: unknown): HostInventory {
         if (inventory[key] !== undefined && (typeof inventory[key] !== 'number' || !Number.isSafeInteger(inventory[key]) || Number(inventory[key]) < 0)) throw new Error(`Invalid inventory ${key}.`);
     }
     if (inventory.cpuUsagePercent !== undefined && (typeof inventory.cpuUsagePercent !== 'number' || !Number.isFinite(inventory.cpuUsagePercent) || inventory.cpuUsagePercent < 0 || inventory.cpuUsagePercent > 100)) throw new Error('Invalid inventory cpuUsagePercent.');
+    validateInstanceResources(inventory);
     for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
     if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
     if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
@@ -253,6 +255,7 @@ export function enrollHost(token: unknown, rawInventory: unknown): { hostId: str
         revokeCredentials(enrollment.host_id);
         const credential = issueCredential(enrollment.host_id);
         database().prepare('UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), now(), inventory.hostname ?? '', JSON.stringify(inventory), enrollment.host_id);
+        recordResources(enrollment.host_id, inventory, now());
         return { hostId: enrollment.host_id, credential, heartbeatSeconds: 15 };
     })();
 }
@@ -284,9 +287,23 @@ export function rotateCredential(hostId: string, nextCredential?: string): strin
 }
 export function heartbeat(hostId: string, value: unknown): Host {
     requireHost(hostId); const inventory = validateInventory(value);
-    database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), inventory.hostname ?? '', JSON.stringify(inventory), hostId);
+    database().transaction(() => {
+        const timestamp = now();
+        database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(timestamp, inventory.hostname ?? '', JSON.stringify(inventory), hostId);
+        recordResources(hostId, inventory, timestamp);
+    })();
     return requireHost(hostId);
 }
+
+function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void {
+    recordInstanceResources(hostId, inventory, timestamp);
+    database().prepare(`INSERT OR REPLACE INTO ec_host_resource_samples
+        (host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,?,?,?,?)`)
+        .run(hostId, timestamp, inventory.cpuCount ?? null, inventory.cpuUsagePercent ?? null,
+            inventory.memoryTotal ?? null, inventory.memoryAvailable ?? null);
+    database().prepare('DELETE FROM ec_host_resource_samples WHERE host_id=? AND sampled_at<?')
+        .run(hostId, new Date(Date.parse(timestamp) - 7 * 86400_000).toISOString());
+}
 export function requireReadyHost(hostId: string, options: { libraryId?: string; requiredBytes?: number; image?: string; port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): Host {
     const host = requireHost(hostId); const reasons = [...host.readinessReasons];
     if (options.requiredBytes !== undefined && (host.inventory.storageAvailable ?? 0) < options.requiredBytes + host.minimumFreeBytes) reasons.push('Insufficient storage for this deployment.');

src/lib/server/hosts/schema.ts

@@ -40,6 +40,25 @@ export function ensureHostSchema(handle: Database.Database): void {
             PRIMARY KEY(host_id,bind_address,protocol,port)
         );
         CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id);
+        CREATE TABLE IF NOT EXISTS ec_host_resource_samples (
+            host_id TEXT NOT NULL REFERENCES ec_hosts(id), sampled_at TEXT NOT NULL,
+            cpu_count REAL, cpu_usage_percent REAL, memory_total REAL, memory_available REAL,
+            PRIMARY KEY(host_id, sampled_at)
+        );
+        CREATE TABLE IF NOT EXISTS ec_instance_resource_samples (
+            host_id TEXT NOT NULL, instance_id TEXT NOT NULL, generation INTEGER NOT NULL,
+            phase TEXT NOT NULL CHECK(phase IN ('startup','running')), bucket TEXT NOT NULL,
+            cpu_cores REAL NOT NULL, memory_bytes REAL NOT NULL,
+            PRIMARY KEY(host_id,instance_id,generation,phase,bucket)
+        );
+        CREATE INDEX IF NOT EXISTS ec_instance_resources_host ON ec_instance_resource_samples(host_id,phase,bucket DESC);
+        CREATE INDEX IF NOT EXISTS ec_instance_resources_fleet ON ec_instance_resource_samples(phase,bucket DESC);
+        CREATE INDEX IF NOT EXISTS ec_instance_resources_retention ON ec_instance_resource_samples(bucket);
+        CREATE TABLE IF NOT EXISTS ec_resource_allowances (
+            scope TEXT NOT NULL, phase TEXT NOT NULL, cpu_cores REAL NOT NULL, memory_bytes REAL NOT NULL,
+            samples INTEGER NOT NULL, window_start TEXT NOT NULL, observed_at TEXT NOT NULL, computed_at TEXT NOT NULL,
+            PRIMARY KEY(scope,phase)
+        );
         CREATE TABLE IF NOT EXISTS ec_host_decommissions (
             host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id),
             operation_id TEXT NOT NULL UNIQUE REFERENCES ec_host_operations(id),

src/lib/server/provisioning/instances.ts

@@ -1,3 +1,4 @@
+import { listHosts } from '../hosts/index.js';
 import { fleetSettings } from '../settings.js';
 import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence.js';
 import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
@@ -176,7 +177,7 @@ interface PreviewRow {
 
 /** Read-only checks; a preview never reserves a slot or writes credentials. */
 export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown; hostId?: unknown; libraryId?: unknown }): LaunchPreview {
-    if (input.hostId !== undefined || !config.komodo.address) return previewNativeCreation(input);
+    if (input.hostId !== undefined || !config.komodo.address || listHosts().length > 0) return previewNativeCreation(input);
     if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.');
     let recipe: LaunchRecipe;
     let blueprintName: string | null = null;
@@ -218,7 +219,7 @@ export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blue
 // releasingServers is internal to atomic collection queueing; HTTP callers cannot set it.
 export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult {
     const nativePreview = input.previewId ? db().prepare("SELECT 1 FROM server_launch_previews WHERE id=? AND json_extract(recipe,'$.native.hostId') IS NOT NULL").get(input.previewId) : null;
-    if (input.hostId || nativePreview || !config.komodo.address) return queueNativeCreation(input);
+    if (input.hostId !== undefined || nativePreview || !config.komodo.address || listHosts().length > 0) return queueNativeCreation(input);
     return db().transaction((): CreateServerResult => {
         const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined;
         if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' };

src/lib/server/provisioning/native.ts

@@ -1,9 +1,11 @@
+import { isIP } from 'node:net';
+import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
 import { randomBytes, randomUUID } from 'node:crypto';
 import { posix } from 'node:path';
 import { db } from '../db/index.js';
 import { config, relDestForKind, serverDir } from '../config.js';
 import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
-import { getHost, getHostOperation, listHosts, nextHostPort, queueHostOperation, releaseHostPorts, requireReadyHost, reserveHostPort } from '../hosts/index.js';
+import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
 import { newestReadyLibrary, requireReadyLibrary } from '../game-files/index.js';
 import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
 import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
@@ -24,14 +26,14 @@ import type { ProvisioningAction } from '../../provisioning.js';
 import { hostAgentUpdatesEnabled } from '../../hosts.js';
 import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
 
-export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }
+export type NativeSelection = import('../../hosts.js').NativePlacement;
 type NativeRecipe = LaunchRecipe & { native?: NativeSelection };
 interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string;
     instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null }
 interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null }
 
-export function nativeSelection(input: Record<string, unknown>, game: string): NativeSelection {
-    if (typeof input.hostId !== 'string' || !input.hostId) throw new Error('Select a ready host.');
+export function nativeSelection(input: Record<string, unknown>, game: string, options: Pick<PlacementOptions, 'planned'> = {}): NativeSelection {
+    if (input.hostId !== undefined && typeof input.hostId !== 'string') throw new Error('Invalid host.');
     const version = newestReadyLibrary(game);
     if (!version) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`);
     if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== version.id) {
@@ -39,14 +41,17 @@ export function nativeSelection(input: Record<string, unknown>, game: string): N
     }
     const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
     if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
-    const host = requireReadyHost(input.hostId, { image, requiredBytes: 1024 ** 3 });
-    requireReadyLibrary(host.id, version.id);
-    const bindAddress = input.bindAddress === undefined || input.bindAddress === '' ? host.bindAddress : input.bindAddress;
-    if (typeof bindAddress !== 'string') throw new Error('Invalid bind address.');
-    if (input.publicAddress !== undefined && (typeof input.publicAddress !== 'string' || input.publicAddress.length > 253)) throw new Error('Invalid public address.');
-    if (input.publicPort !== undefined && (!Number.isInteger(input.publicPort) || Number(input.publicPort) < 1 || Number(input.publicPort) > 65535)) throw new Error('Invalid public port.');
-    return { hostId: host.id, libraryId: version.id, image, bindAddress, publicAddress: input.publicAddress as string | undefined ?? host.publicAddress,
-        publicPort: input.publicPort as number | undefined };
+    const automatic = input.automatic === true || !input.hostId;
+    const rawNetwork = input.automatic === true ? input.network ?? {} : input;
+    if (!rawNetwork || typeof rawNetwork !== 'object' || Array.isArray(rawNetwork)) throw new Error('Invalid network overrides.');
+    const network = rawNetwork as NativeSelection;
+    const bindAddress = network.bindAddress || undefined;
+    if (bindAddress !== undefined && (typeof bindAddress !== 'string' || !isIP(bindAddress))) throw new Error('Invalid bind address.');
+    const { host } = choosePlacement({ hostId: automatic ? undefined : input.hostId as string, libraryId: version.id, image, bindAddress, ...options });
+    if (network.publicAddress !== undefined && (typeof network.publicAddress !== 'string' || network.publicAddress.length > 253)) throw new Error('Invalid public address.');
+    if (network.publicPort !== undefined && (!Number.isInteger(network.publicPort) || Number(network.publicPort) < 1 || Number(network.publicPort) > 65535)) throw new Error('Invalid public port.');
+    return { hostId: host.id, libraryId: version.id, image, bindAddress: bindAddress ?? host.bindAddress, publicAddress: network.publicAddress ?? host.publicAddress,
+        publicPort: network.publicPort, automatic, network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } };
 }
 
 export function previewNativeCreation(input: Record<string, unknown>): LaunchPreview {
@@ -72,10 +77,10 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
     db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)')
         .run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
     const checks: LaunchPreview['checks'] = [
-        { label: 'Host readiness', state: 'pass', detail: `${getHost(selection.hostId)!.name}: connected, Docker ready, storage and runtime verified.` },
-        { label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on the selected host.` },
+        { label: 'Host capacity', state: 'pass', detail: selection.automatic ? 'Capacity available. The least loaded eligible host is selected when provisioning starts.' : `${getHost(selection.hostId)!.name}: capacity available.` },
+        { label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on an eligible host.` },
         { label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' },
-        { label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
+        { label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? selection.automatic ? 'UDP/TCP capacity available; port assigned at placement.' : `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
         { label: 'Configuration', state: 'pass', detail: recipe.configSource }
     ];
     return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, checks,
@@ -83,16 +88,9 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
 }
 
 export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null {
-    const excluded = [...taken];
-    for (;;) {
-        const port = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'udp');
-        if (port === null) return null;
-        // Both protocols are published by Plutainer and reserved together.
-        const tcp = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'tcp');
-        if (tcp === port) return port;
-        if (tcp !== null && tcp < port) { excluded.push(tcp); continue; }
-        excluded.push(port);
-    }
+    const host = getHost(selection.hostId);
+    return host ? availableHostPort(host, { bindAddress: selection.bindAddress,
+        planned: taken.map(port => ({ hostId: host.id, port })) }) : null;
 }
 
 export function queueNativeCreation(input: CreateServerInput): CreateServerResult {
@@ -113,7 +111,7 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
             const name = `${game}_zombie_${input.mapSlug}_${lobby}`;
             const port = nextNativePort(selection);
             if (!port) throw new Error('No available host port.');
-            if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || preview.port !== port))) throw new Error('Available lobby or host port changed. Review a fresh preview.');
+            if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || (!selection.automatic && preview.port !== port)))) throw new Error('Available lobby or host port changed. Review a fresh preview.');
             const key = input.plutoKey?.trim() || getServerKey(name);
             if (!key) throw new Error(`Save a Plutonium server key for ${name}.`);
             saveServerKey(name, key);
@@ -128,11 +126,11 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
             db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
                 .run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
             db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null);
-            if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=? WHERE id=?').run(id, preview.id);
+            if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id);
             deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' });
             return { ok: true as const, name, port, deploymentId: id };
         })();
-    } catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
+    } catch (error) { return { ok: false, kind: error instanceof CapacityError ? 'conflict' : 'bad-request', message: (error as Error).message }; }
 }
 
 export function bindNativeOperation(id: string, server: string): void {
@@ -270,12 +268,43 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
 
 const activeJobs = new Map<string, Promise<void>>();
 
+/** Reservations can move only while no remote work has ever been submitted. */
+function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance {
+    return db().transaction(() => {
+        const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe;
+        const prior = recipe.native!;
+        const network = prior.automatic ? prior.network ?? {} : prior;
+        const { host, port } = choosePlacement({ hostId: prior.automatic ? undefined : instance.host_id,
+            libraryId: instance.library_id, image: instance.image, bindAddress: network.bindAddress,
+            thorough: true, excludeInstanceId: instance.id });
+        const bind = network.bindAddress ?? host.bindAddress;
+        releaseHostPorts(instance.id);
+        reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'udp' });
+        reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'tcp' });
+        db().prepare(`UPDATE native_instances SET host_id=?,bind_address=?,port=?,public_address=?,public_port=? WHERE id=?`)
+            .run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id);
+        db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id);
+        recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort };
+        db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
+        db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id);
+        deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' });
+        return nativeInstance(job.server)!;
+    })();
+}
+
 async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     if (!stillActive(job)) return;
     let instance = nativeInstance(job.server);
     if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
         finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
     }
+    if (job.action === 'create' && !job.operation_id && !job.stage_operation_id && !job.submitted_at) {
+        const settings = fleetSettings();
+        const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
+            AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
+        if (active.n >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
+        instance = placeQueuedCreation(job, instance);
+    }
     const host = getHost(instance.host_id);
     if (!host?.connected || host.mode === 'disabled') {
         db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server);
@@ -403,7 +432,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     }
     if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
     deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
-    if (job.action === 'create') requireReadyHost(instance.host_id, { image: instance.image, requiredBytes: 1024 ** 3 });
+    if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
+        bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true });
     if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id);
     if (job.action !== 'create' && job.action !== 'start') {
         // Fast controller check keeps occupied lobbies out of the operation slots.