From e6cdf9eb7f17e60fa44e2e22b6424a88eb5063e2 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Sun, 4 Oct 2026 21:04:09 -0400 Subject: [PATCH] Automatically place servers using learned resource capacity --- README.md | 8 +- agent/ec_agent.py | 62 +++++- agent/tests/test_agent.py | 44 +++++ docs/native-hosts.md | 56 +++++- src/lib/deployment-plan.ts | 2 +- src/lib/hosts.ts | 19 +- src/lib/server/backends/schema.ts | 2 + src/lib/server/fleet/collections.ts | 22 ++- src/lib/server/fleet/launch-recipes.ts | 2 +- src/lib/server/hosts/calibration.ts | 106 ++++++++++ src/lib/server/hosts/capacity.ts | 83 ++++++++ src/lib/server/hosts/index.ts | 19 +- src/lib/server/hosts/schema.ts | 19 ++ src/lib/server/provisioning/instances.ts | 5 +- src/lib/server/provisioning/native.ts | 88 ++++++--- .../components/DeployCollectionDialog.svelte | 14 +- .../ui/components/DeployServerDialog.svelte | 16 +- .../ui/components/NativeHostSelection.svelte | 20 +- src/routes/api/hosts/[id]/+server.ts | 3 +- src/routes/api/servers/+server.ts | 2 +- src/routes/hosts/[id]/+page.svelte | 13 +- static/agent/ec-agent.py | 93 ++++++++- static/agent/manifest.json | 4 +- tests/unit/host-capacity-calibration.test.ts | 129 +++++++++++++ tests/unit/native-provisioning.test.ts | 181 +++++++++++++++++- 25 files changed, 927 insertions(+), 85 deletions(-) create mode 100644 src/lib/server/hosts/calibration.ts create mode 100644 src/lib/server/hosts/capacity.ts create mode 100644 tests/unit/host-capacity-calibration.test.ts diff --git a/README.md b/README.md index fb3b067..06bec90 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,11 @@ including SQLite metadata and published game-file storage. shows setup progress. Automatic agent updates are enabled by default. 3. Add the exact Plutonium server keys in **Keys**, or supply missing keys when launching. -4. Launch a server or collection on a host. The latest files for each game and - the default runtime are selected automatically. EC reserves host-specific - ports and verifies startup. +4. Launch a server or collection. EC quickly checks CPU, memory and port capacity, + then automatically places each server on the least loaded eligible host. The + latest game files and default runtime are selected automatically. EC reserves + resources for queued launches and verifies startup. CPU/memory allowances learn + from actual container usage, using temporary defaults while calibrating. See [Native host management](docs/native-hosts.md) for installation, credentials, storage, failure recovery, API contracts and the live-host acceptance checklist. diff --git a/agent/ec_agent.py b/agent/ec_agent.py index 57a00e6..872eddb 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -9,6 +9,8 @@ import argparse import base64 from concurrent.futures import ThreadPoolExecutor, as_completed import contextlib +from datetime import datetime, timezone +import math import hashlib import fcntl import json @@ -30,7 +32,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.4.2" +VERSION = "1.4.3" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 LIBRARY_DOWNLOAD_WORKERS = 4 @@ -929,6 +931,58 @@ class Agent: return None return round(100 * (total - deltas[3] - deltas[4]) / total, 1) + def instance_resources(self): + """One bounded Docker stats batch; unavailable samples are never zero load.""" + ids = self.docker.call("container", "ls", "--filter", LABEL + ".managed=true", + "--no-trunc", "--format", "{{.ID}}", timeout=10).splitlines()[:1000] + if not ids: + return [] + containers = json.loads(self.docker.call("container", "inspect", *ids, timeout=10)) + owned = {} + for info in containers: + try: + labels = info.get("Config", {}).get("Labels", {}) or {} + if labels.get(LABEL + ".managed") != "true" or not info.get("State", {}).get("Running"): + continue + instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", ""))) + name = self.container_name(instance_id, generation) + if info.get("Name", "").lstrip("/") != name: + continue + expected = str(self.instance_dir(instance_id, generation) / "app") + if not any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", [])): + continue + started = datetime.fromisoformat(info["State"]["StartedAt"].replace("Z", "+00:00")) + uptime = max(0, int((datetime.now(timezone.utc) - started).total_seconds())) + owned[name] = {"instanceId": instance_id, "generation": generation, "uptimeSeconds": uptime} + except (AgentError, ValueError, KeyError, TypeError): + continue + if not owned: + return [] + rows = self.docker.call("stats", "--no-stream", "--no-trunc", "--format", "{{json .}}", *owned, timeout=15) + samples = [] + units = {"B": 1, "KiB": 1024, "MiB": 1024 ** 2, "GiB": 1024 ** 3, + "TiB": 1024 ** 4, "kB": 1000, "MB": 1000 ** 2, "GB": 1000 ** 3} + for line in rows.splitlines(): + try: + row = json.loads(line) + identity = owned.get(row.get("Name")) + if not identity or int(row["PIDs"]) <= 0: + continue + # Docker Linux CPU percent is core-based: 150% means 1.5 cores. + percent = row["CPUPerc"] + if not percent.endswith("%"): + continue + cores = float(percent[:-1]) / 100 + used = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)", row["MemUsage"].split("/", 1)[0].strip()) + if not used or used[2] not in units or not math.isfinite(cores) or cores < 0: + continue + memory = math.ceil(float(used[1]) * units[used[2]]) + if memory > 0: + samples.append({**identity, "cpuCores": cores, "memoryBytes": memory}) + except (ValueError, KeyError, TypeError, AttributeError, OverflowError): + continue + return samples + def inventory(self): cpu_usage = self.cpu_usage_percent() usage = shutil.disk_usage(self.root) @@ -1028,6 +1082,10 @@ class Agent: if aliases.exists(): with contextlib.suppress(OSError, ValueError): images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids) + resources = {} + if docker_running: + with contextlib.suppress(AgentError, ValueError, TypeError, KeyError): + resources["instanceResources"] = self.instance_resources() os_name = platform.system() with contextlib.suppress(OSError): for line in Path("/etc/os-release").read_text().splitlines(): @@ -1035,7 +1093,7 @@ class Agent: os_name = line.split("=", 1)[1].strip('"') update = self.update_inventory() capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) - return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} + return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} def safe_to_disrupt(self, instance_id, generation): health = self.inspect(instance_id, generation) diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 01bc235..0c4a4d5 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -107,6 +107,50 @@ class AgentTests(unittest.TestCase): else: self.assertEqual(inventory["cpuUsagePercent"], usage) + def resource_container(self): + return {"Name": self.agent.container_name(INSTANCE, 1), + "Config": {"Labels": {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1"}}, + "State": {"Running": True, "StartedAt": "2000-01-01T00:00:00.000000000Z"}, + "Mounts": [{"Type": "bind", "Source": str(self.agent.instance_dir(INSTANCE, 1) / "app"), "Destination": "/home/plutainer/app"}]} + + def test_instance_resources_batches_owned_containers_and_normalizes_core_cpu_and_memory(self): + name = self.agent.container_name(INSTANCE, 1) + rows = json.dumps({"Name": name, "CPUPerc": "150.00%", "MemUsage": "384.5MiB / 8GiB", "PIDs": "12"}) + with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([self.resource_container()]), rows]) as call: + samples = self.agent.instance_resources() + self.assertEqual(len(samples), 1) + self.assertEqual(samples[0]["instanceId"], INSTANCE) + self.assertEqual(samples[0]["generation"], 1) + self.assertEqual(samples[0]["cpuCores"], 1.5) + self.assertEqual(samples[0]["memoryBytes"], int(384.5 * 1024 ** 2)) + self.assertGreater(samples[0]["uptimeSeconds"], 300) + self.assertEqual(call.call_args_list[-1].args, ("stats", "--no-stream", "--no-trunc", "--format", "{{json .}}", name)) + + def test_instance_resources_ignores_stopped_foreign_and_mismatched_containers(self): + for change in ("stopped", "foreign", "generation", "name"): + info = self.resource_container() + if change == "stopped": + info["State"]["Running"] = False + elif change == "foreign": + info["Mounts"][0]["Source"] = "/another-installation/app" + elif change == "generation": + info["Config"]["Labels"][module.LABEL + ".generation"] = "2" + else: + info["Name"] = "unrelated" + with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([info])]) as call: + self.assertEqual(self.agent.instance_resources(), []) + self.assertEqual(call.call_count, 2) + + def test_instance_resources_skips_unavailable_stats_instead_of_reporting_zero_usage(self): + good = {"Name": self.agent.container_name(INSTANCE, 1), "CPUPerc": "0.00%", "MemUsage": "200MiB / 8GiB", "PIDs": "10"} + for override in ({"CPUPerc": "NaN%"}, {"CPUPerc": "-1%"}, {"CPUPerc": "--"}, {"MemUsage": "bad"}, {"MemUsage": "0B / 0B"}, {"PIDs": "0"}, {"Name": "foreign"}): + with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([self.resource_container()]), json.dumps({**good, **override})]): + self.assertEqual(self.agent.instance_resources(), []) + with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([self.resource_container()]), json.dumps(good)]): + self.assertEqual(self.agent.instance_resources()[0]["cpuCores"], 0) + with patch.object(self.agent, "instance_resources", side_effect=AgentError("timed out")): + self.assertNotIn("instanceResources", self.agent.inventory()) + def operation(self, kind, payload=None, op_id="operation-1", generation=1): return {"id": op_id, "kind": kind, "instanceId": INSTANCE, "generation": generation, "payload": payload or {}} diff --git a/docs/native-hosts.md b/docs/native-hosts.md index 95e9793..da289e3 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -27,8 +27,8 @@ the agent owns local execution and its durable operation journal. progress and errors. Wait for verification before launching. Hosts report connection and readiness separately; deployment also checks the particular game files, image, capacity and ports required for the launch. -6. Launch a server or collection. Select the host; the latest game files and - default runtime are selected automatically. An advanced runtime override is +6. Launch a server or collection. The host, latest game files and default + runtime are selected automatically after checking capacity. An advanced runtime override is available. If the latest files change after previewing, refresh the preview before launching. EC selects lobby IDs and reserves UDP/TCP ports atomically. @@ -121,7 +121,57 @@ host-wide busy time across all cores on a 0–100% scale between inventory sampl (normally about 15 seconds). Idle and I/O wait are excluded; stolen CPU time counts as unavailable capacity. The first sample after startup, unavailable counters, and older agents show **Not reported**. Offline hosts retain their -last reported value. This is current inventory, not historical metrics. +last reported value. The controller retains CPU and memory samples for seven days (missing measurements +remain unknown). Scheduling uses the last five minutes as well as the latest +heartbeat; the Inventory panel continues to show the latest reading. + +New servers are placed automatically. Preview and submission perform local +capacity checks without contacting agents: a connected, active host must have +verified game files, its runtime, storage, a free UDP/TCP port, and valid CPU and +memory measurements. Hosts with stale or missing measurements cannot accept new +servers. Existing API clients may still explicitly constrain a launch to a host. + +Admission reserves a provisional host and port atomically, including collection +launches. Agent **1.4.3** samples each managed running container with a bounded +[`docker stats`](https://docs.docker.com/reference/cli/docker/container/stats/) +batch during inventory collection. CPU is normalized to cores (100% Docker CPU +is one core); memory is the Linux working usage reported by Docker, excluding +reclaimable cache. Missing or failed samples stay unknown instead of becoming +zero usage. Samples must match the host's live instance and generation. + +The controller retains per-instance minute peaks for 24 hours and caches learned +CPU/memory allowances. The first five minutes after a container starts count as +startup; later samples train the running allowance. Startup uses the observed +maximum and running uses the 95th percentile of minute peaks, both with **25% +headroom**. Each calculation considers up to the latest 10,000 minute peaks. +Host-local evidence is preferred to account for different hardware; new hosts +can borrow the fleet's learned allowance. Startup allowances are never lower +than running allowances. The host page shows both allowances and their source. + +Lower allowances require at least three startup minute samples spanning two +minutes, or twenty running minute samples spanning five minutes. Until then, +**0.5 core and 1 GiB** are temporary cold-start allowances, raised immediately +if observations exceed them. They are replaced by measured allowances once the +evidence threshold is met; a tiny 0.01-core floor prevents zero-CPU admission. +Models expire after 24 hours without measurements. Recalculation happens during +heartbeat ingestion, at most once per minute per host/fleet phase; new peaks +raise the cache immediately. Launch admission reads only the cached model. + +Scheduling still leaves **10% host CPU and 512 MiB RAM** for the OS and bursts. +These allowances are scheduling reservations, not Docker limits. Queued and +unresolved creations consume the startup allowance; established instances keep +the running allowance even while idle or stopped. Observed host load also counts, +so other processes and previously unseen load increases reduce available capacity. + +When startup capacity is available, provisioning rechecks all eligible hosts and +chooses the lowest projected CPU/memory utilization, using the busier of the +latest reading and the five-minute average. It moves the provisional reservation +atomically and records the final host in the launch recipe. Host network defaults +are resolved again for the final host; explicit overrides are preserved. If no +host qualifies, the request waits and is checked again. Once remote staging or +creation has been submitted, placement stays fixed to prevent duplicate instances. +Cancelling an unsubmitted launch releases its reservation; uncertain remote work +retains capacity until cleanup is confirmed. When publishing changed agent code, bump its numeric `VERSION` in `agent/ec_agent.py` and rebuild the controller. Packaging derives the manifest diff --git a/src/lib/deployment-plan.ts b/src/lib/deployment-plan.ts index d853146..dee7669 100644 --- a/src/lib/deployment-plan.ts +++ b/src/lib/deployment-plan.ts @@ -45,7 +45,7 @@ export interface CollectionPreview { removals: { name: string; reason: string }[]; entries: { count: number; existing: number; missing: number; existingServers: string[]; coverage: { name: string; health: string; href: string }[]; recipe: LaunchRecipeSummary }[]; - servers: { entryIndex: number; name: string | null; port: number | null; hasSavedKey: boolean }[]; + servers: { native?: import('./hosts.js').NativePlacement; entryIndex: number; name: string | null; port: number | null; hasSavedKey: boolean }[]; blockers: string[]; ready: boolean; } diff --git a/src/lib/hosts.ts b/src/lib/hosts.ts index d913217..2844e28 100644 --- a/src/lib/hosts.ts +++ b/src/lib/hosts.ts @@ -68,7 +68,16 @@ export interface HostInventory { currentOperation?: string | null; [key: string]: unknown; capabilities?: string[]; agentUpdate?: HostAgentUpdate; + instanceResources?: InstanceResources[]; } +export interface InstanceResources { + instanceId: string; generation: number; uptimeSeconds: number; cpuCores: number; memoryBytes: number; +} +export interface ResourceAllowance { + cpuCores: number; memoryBytes: number; source: 'host' | 'fleet' | 'bootstrap'; + samples: number; observedAt: string | null; +} +export interface CapacityCalibration { running: ResourceAllowance; startup: ResourceAllowance } export interface Host { id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number; bindAddress: string; publicAddress: string; minimumFreeBytes: number; createdAt: string; @@ -95,6 +104,14 @@ export interface HostPreparation { } export interface NativeDeploymentSelection { - hostId: string; libraryIds: Partial>; image: string; + libraryIds: Partial>; image: string; bindAddress?: string; publicAddress?: string; publicPort?: number; } + +/** The host/port are provisional until the creation command is submitted. */ +export interface NativePlacement { + hostId: string; libraryId: string; image?: string; + bindAddress?: string; publicAddress?: string; publicPort?: number; + automatic?: boolean; + network?: { bindAddress?: string; publicAddress?: string; publicPort?: number }; +} diff --git a/src/lib/server/backends/schema.ts b/src/lib/server/backends/schema.ts index 1926131..f200041 100644 --- a/src/lib/server/backends/schema.ts +++ b/src/lib/server/backends/schema.ts @@ -9,6 +9,7 @@ export function migrateNativeInstances(handle: Database.Database): void { port INTEGER NOT NULL, public_address TEXT, public_port INTEGER, created_at TEXT NOT NULL, retired_at TEXT ); + CREATE INDEX IF NOT EXISTS native_instance_host ON native_instances(host_id) WHERE retired_at IS NULL; CREATE UNIQUE INDEX IF NOT EXISTS native_instance_live_name ON native_instances(server) WHERE retired_at IS NULL; CREATE TABLE IF NOT EXISTS native_provisioning ( deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id), @@ -16,6 +17,7 @@ export function migrateNativeInstances(handle: Database.Database): void { stage_operation_id TEXT, operation_id TEXT, files_json TEXT, applied_at TEXT, verified_at TEXT ); + CREATE INDEX IF NOT EXISTS native_provisioning_instance ON native_provisioning(instance_id); CREATE TABLE IF NOT EXISTS native_creation_cancellations ( creation_id TEXT PRIMARY KEY REFERENCES server_deployments(id), cleanup_id TEXT NOT NULL UNIQUE REFERENCES server_deployments(id) diff --git a/src/lib/server/fleet/collections.ts b/src/lib/server/fleet/collections.ts index 8a2fb8a..2bf8335 100644 --- a/src/lib/server/fleet/collections.ts +++ b/src/lib/server/fleet/collections.ts @@ -12,6 +12,7 @@ import { serverObservation } from './status.js'; import { activeBetaRemovalBlocker, getServer } from './servers.js'; import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js'; import { nativeSelection, nextNativePort } from '../provisioning/native.js'; +import { listHosts } from '../hosts/index.js'; import { nativeInstance } from '../backends/native.js'; function coverage(entries: BlueprintEntry[]) { @@ -73,15 +74,24 @@ function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode) { function allocate(entries: BlueprintEntry[], releasingServers: string[] = []): CollectionPreview['servers'] { const names: string[] = []; const ports: number[] = []; + const planned: { hostId: string; port: number }[] = []; const counts = coverage(entries); return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => { const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`; const lobby = nextLobbyNumber(base, names); const name = lobby === null ? null : `${base}_${lobby}`; - const port = entry.recipe.native ? nextNativePort(entry.recipe.native, ports) : nextAvailablePort(ports, releasingServers); + let native = entry.recipe.native; + let port: number | null; + if (native) { + try { + native = nativeSelection(native as unknown as Record, entry.recipe.game, { planned }); + port = nextNativePort(native, planned.filter(item => item.hostId === native!.hostId).map(item => item.port)); + if (port !== null) planned.push({ hostId: native.hostId, port }); + } catch { port = null; } + } else port = nextAvailablePort(ports, releasingServers); if (name) names.push(name); if (port !== null) ports.push(port); - return { entryIndex, name, port, hasSavedKey: !!name && !!getServerKey(name) }; + return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && !!getServerKey(name) }; })); } @@ -98,7 +108,7 @@ function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEn if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.'); } if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.'); - if (servers.some(server => server.port === null)) blockers.push(`There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`); + if (servers.some(server => server.port === null)) blockers.push(entries.some(entry => entry.recipe.native) ? 'There is not enough host CPU, memory or port capacity for the whole collection.' : `There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`); return blockers; } @@ -117,7 +127,7 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill', if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.'); return db().transaction((): CollectionPreview => { const { entries, blueprintName } = blueprintEntries(blueprintId); - if (selection?.hostId !== undefined || !config.komodo.address) { + if (selection?.hostId !== undefined || !config.komodo.address || listHosts().length > 0) { const libraries = selection?.libraryIds as Record | undefined; for (const entry of entries) { entry.recipe.native = nativeSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game); @@ -159,7 +169,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): const servers = allocate(entries, releasingServers); const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)]; if (blockers.length) throw new Error(blockers.join(' ')); - if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || server.port !== reviewed[i]?.port)) { + if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || (!server.native?.automatic && server.port !== reviewed[i]?.port))) { throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.'); } const expected = new Set(servers.map(server => server.name)); @@ -196,7 +206,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): return { id, ports }; }); servers.forEach((server, index) => { - const recipe = entries[server.entryIndex].recipe; + const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) }; const childPreviewId = randomUUID(); db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)') .run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name); diff --git a/src/lib/server/fleet/launch-recipes.ts b/src/lib/server/fleet/launch-recipes.ts index 2902498..23d9f6e 100644 --- a/src/lib/server/fleet/launch-recipes.ts +++ b/src/lib/server/fleet/launch-recipes.ts @@ -9,7 +9,7 @@ import { getBaselineForScope, getVersion, listPromotedOverlaysForScope } from '. import { sha256Buffer } from './files.js'; export interface LaunchRecipe { - native?: { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }; + native?: import('../../hosts.js').NativePlacement; format: 1; game: ZombieGame; mapSlug: string; diff --git a/src/lib/server/hosts/calibration.ts b/src/lib/server/hosts/calibration.ts new file mode 100644 index 0000000..c2726c0 --- /dev/null +++ b/src/lib/server/hosts/calibration.ts @@ -0,0 +1,106 @@ +import { db } from '../db/index.js'; +import { ensureHostSchema } from './schema.js'; +import type { CapacityCalibration, HostInventory, ResourceAllowance } from '../../hosts.js'; + +const BOOTSTRAP_CPU = 0.5; +const BOOTSTRAP_MEMORY = 1024 ** 3; +const MARGIN = 1.25; +const WINDOW_MS = 24 * 60 * 60_000; +const MAX_SAMPLES = 10_000; +type Phase = 'startup' | 'running'; +interface AllowanceRow { + scope: string; phase: Phase; cpu_cores: number; memory_bytes: number; samples: number; + window_start: string; observed_at: string; computed_at: string; +} +function database() { const handle = db(); ensureHostSchema(handle); return handle; } + +export function validateInstanceResources(inventory: HostInventory): void { + const samples = inventory.instanceResources; + if (samples === undefined) return; + if (!Array.isArray(samples) || samples.length > 1000) throw new Error('Invalid instance resource samples.'); + const seen = new Set(); + for (const sample of samples) { + if (!sample || typeof sample !== 'object' || typeof sample.instanceId !== 'string' || !sample.instanceId || sample.instanceId.length > 100 + || !Number.isSafeInteger(sample.generation) || sample.generation < 1 + || !Number.isSafeInteger(sample.uptimeSeconds) || sample.uptimeSeconds < 0 + || typeof sample.cpuCores !== 'number' || !Number.isFinite(sample.cpuCores) || sample.cpuCores < 0 + || !Number.isSafeInteger(sample.memoryBytes) || sample.memoryBytes <= 0 + || (inventory.cpuCount !== undefined && sample.cpuCores > inventory.cpuCount + 0.01) + || (inventory.memoryTotal !== undefined && sample.memoryBytes > inventory.memoryTotal) + || seen.has(sample.instanceId)) throw new Error('Invalid instance resource sample.'); + seen.add(sample.instanceId); + } +} + +/** Heartbeat ingestion owns all learning work. Launch requests only read cached allowances. */ +export function recordInstanceResources(hostId: string, inventory: HostInventory, timestamp: string): void { + if (!inventory.instanceResources?.length || !inventory.dockerRunning || !inventory.cpuCount || !inventory.memoryTotal) return; + const handle = database(); + const bucket = timestamp.slice(0, 16) + ':00.000Z'; + const observed = new Map(); + const owned = handle.prepare('SELECT 1 FROM native_instances WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL'); + const insert = handle.prepare(`INSERT INTO ec_instance_resource_samples + (host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,?,?,?,?,?,?) + ON CONFLICT(host_id,instance_id,generation,phase,bucket) DO UPDATE SET + cpu_cores=MAX(cpu_cores,excluded.cpu_cores), memory_bytes=MAX(memory_bytes,excluded.memory_bytes)`); + for (const sample of inventory.instanceResources) { + // An agent can only train from its currently owned generation; foreign, + // retired and delayed pre-rebuild samples cannot contaminate the model. + if (!owned.get(sample.instanceId, hostId, sample.generation)) continue; + const phase: Phase = sample.uptimeSeconds < 300 ? 'startup' : 'running'; + insert.run(hostId, sample.instanceId, sample.generation, phase, bucket, sample.cpuCores, sample.memoryBytes); + const prior = observed.get(phase); + observed.set(phase, { cpu: Math.max(prior?.cpu ?? 0, sample.cpuCores), memory: Math.max(prior?.memory ?? 0, sample.memoryBytes) }); + } + for (const [phase, peak] of observed) for (const scope of [hostId, '*']) { + const previous = handle.prepare('SELECT * FROM ec_resource_allowances WHERE scope=? AND phase=?').get(scope, phase) as AllowanceRow | undefined; + if (!previous || previous.computed_at < bucket) { + const rows = handle.prepare(`SELECT cpu_cores, memory_bytes, bucket FROM ec_instance_resource_samples + WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`) + .all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as + { cpu_cores: number; memory_bytes: number; bucket: string }[]; + const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1]; + const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu) * MARGIN; + const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory) * MARGIN); + handle.prepare(`INSERT INTO ec_resource_allowances + (scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(scope,phase) DO UPDATE SET + cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples, + window_start=excluded.window_start,observed_at=excluded.observed_at,computed_at=excluded.computed_at`) + .run(scope, phase, cpu, memory, rows.length, rows.at(-1)!.bucket, timestamp, timestamp); + if (scope === '*') handle.prepare('DELETE FROM ec_instance_resource_samples WHERE bucket=?') + .all(hostId, '*', new Date(Date.now() - WINDOW_MS).toISOString()) as AllowanceRow[]; + const allowance = (phase: Phase): ResourceAllowance => { + const local = rows.find(row => row.scope === hostId && row.phase === phase); + const fleet = rows.find(row => row.scope === '*' && row.phase === phase); + const learned = (row?: AllowanceRow) => !!row && row.samples >= (phase === 'startup' ? 3 : 20) + && Date.parse(row.observed_at) - Date.parse(row.window_start) >= (phase === 'startup' ? 120_000 : 300_000); + const selected = learned(local) ? local : learned(fleet) ? fleet : undefined; + return { + // A few higher observations can increase a fleet/bootstrap allowance + // immediately. Decreasing it requires the evidence thresholds above. + cpuCores: Math.max(0.01, selected?.cpu_cores ?? BOOTSTRAP_CPU, local?.cpu_cores ?? 0, selected ? 0 : fleet?.cpu_cores ?? 0), + memoryBytes: Math.max(selected?.memory_bytes ?? BOOTSTRAP_MEMORY, local?.memory_bytes ?? 0, selected ? 0 : fleet?.memory_bytes ?? 0), + source: selected ? selected.scope === hostId ? 'host' : 'fleet' : 'bootstrap', + samples: selected?.samples ?? local?.samples ?? fleet?.samples ?? 0, + observedAt: selected?.observed_at ?? local?.observed_at ?? fleet?.observed_at ?? null + }; + }; + const running = allowance('running'); const startup = allowance('startup'); + if (running.source === 'bootstrap' && (running.cpuCores > startup.cpuCores || running.memoryBytes > startup.memoryBytes)) startup.source = 'bootstrap'; + startup.cpuCores = Math.max(startup.cpuCores, running.cpuCores); + startup.memoryBytes = Math.max(startup.memoryBytes, running.memoryBytes); + return { running, startup }; +} diff --git a/src/lib/server/hosts/capacity.ts b/src/lib/server/hosts/capacity.ts new file mode 100644 index 0000000..63aa277 --- /dev/null +++ b/src/lib/server/hosts/capacity.ts @@ -0,0 +1,83 @@ +import { db } from '../db/index.js'; +import { listHosts, requireReadyHost } from './index.js'; +import { requireReadyLibrary } from '../game-files/index.js'; +import type { Host } from '../../hosts.js'; +import { capacityCalibration } from './calibration.js'; + +// Storage and OS headroom are separate from learned instance CPU/memory usage. +const INSTANCE_STORAGE_BYTES = 1024 ** 3; +const MEMORY_HEADROOM = 512 * 1024 ** 2; +export class CapacityError extends Error {} +export interface PlacementOptions { + hostId?: string; libraryId: string; image: string; bindAddress?: string; + thorough?: boolean; excludeInstanceId?: string; + planned?: { hostId: string; port: number }[]; +} + +/** One indexed port read per host, including both protocols and wildcard binds. */ +export function availableHostPort(host: Host, options: Pick = {}): number | null { + const bind = options.bindAddress ?? host.bindAddress; + const rows = db().prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?') + .all(host.id, options.excludeInstanceId ?? '') as { port: number; bind_address: string }[]; + const used = new Set(rows.filter(row => row.bind_address === bind || ['0.0.0.0', '::'].includes(bind) + || ['0.0.0.0', '::'].includes(row.bind_address)).map(row => row.port)); + for (const item of options.planned ?? []) if (item.hostId === host.id) used.add(item.port); + for (let port = host.portMin; port <= host.portMax; port++) if (!used.has(port)) return port; + return null; +} + +function projectedLoad(host: Host, options: PlacementOptions): number { + const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory; + if (!cpuCount || cpuUsagePercent === undefined || !memoryTotal || memoryAvailable === undefined || memoryAvailable > memoryTotal) { + throw new Error('Waiting for valid CPU and memory measurements.'); + } + let cpu = cpuUsagePercent; + let available = memoryAvailable; + if (options.thorough) { + const history = db().prepare(`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory + FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`) + .get(host.id, new Date(Date.now() - 5 * 60_000).toISOString(), cpuCount, memoryTotal) as { cpu: number | null; memory: number | null }; + cpu = Math.max(cpu, history.cpu ?? cpu); + available = Math.min(available, history.memory ?? available); + } + // Unfinished/uncertain launches remain charged until recovery and a later + // heartbeat. Observed usage accounts for established servers; the count + // floor also reserves their minimum footprint while they are idle/stopped. + const instances = db().prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(NOT EXISTS ( + SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id + WHERE n.instance_id=i.id AND d.action='create' AND d.state IN ('done','warning') AND d.finished_at item.hostId === host.id).length ?? 0; + const count = instances.total + planned + 1; + const pending = instances.pending + planned + 1; + const { running, startup } = capacityCalibration(host.id); + const cores = Math.max(cpuCount * cpu / 100 + pending * startup.cpuCores, + (count - pending) * running.cpuCores + pending * startup.cpuCores); + const memory = Math.max(memoryTotal - available + pending * startup.memoryBytes, + (count - pending) * running.memoryBytes + pending * startup.memoryBytes); + if (cores > cpuCount * 0.9) throw new Error('Insufficient CPU capacity.'); + if (memory > memoryTotal - MEMORY_HEADROOM) throw new Error('Insufficient memory capacity.'); + return Math.max(cores / cpuCount, memory / memoryTotal); +} + +/** Admission stops at the first fit; placement evaluates and ranks every fit. No remote I/O. */ +export function choosePlacement(options: PlacementOptions): { host: Host; port: number; load: number } { + const candidates: { host: Host; port: number; load: number }[] = []; + const reasons: string[] = []; + for (const host of listHosts().filter(host => !options.hostId || host.id === options.hostId)) { + try { + requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES }); + requireReadyLibrary(host.id, options.libraryId); + const port = availableHostPort(host, options); + if (port === null) throw new Error('No available UDP/TCP game port.'); + const load = projectedLoad(host, options); + const candidate = { host, port, load }; + if (!options.thorough) return candidate; + candidates.push(candidate); + } catch (error) { reasons.push(`${host.name}: ${(error as Error).message}`); } + } + candidates.sort((a, b) => a.load - b.load || a.host.id.localeCompare(b.host.id)); + if (candidates.length) return candidates[0]; + throw new CapacityError(`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`); +} diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts index 460bad8..d390139 100644 --- a/src/lib/server/hosts/index.ts +++ b/src/lib/server/hosts/index.ts @@ -4,6 +4,7 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypt import { isIP } from 'node:net'; import { db } from '../db/index.js'; import { ensureHostSchema } from './schema.js'; +import { recordInstanceResources, validateInstanceResources } from './calibration.js'; import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js'; export const AGENT_PROTOCOL_VERSION = 1; @@ -219,6 +220,7 @@ export function validateInventory(value: unknown): HostInventory { if (inventory[key] !== undefined && (typeof inventory[key] !== 'number' || !Number.isSafeInteger(inventory[key]) || Number(inventory[key]) < 0)) throw new Error(`Invalid inventory ${key}.`); } if (inventory.cpuUsagePercent !== undefined && (typeof inventory.cpuUsagePercent !== 'number' || !Number.isFinite(inventory.cpuUsagePercent) || inventory.cpuUsagePercent < 0 || inventory.cpuUsagePercent > 100)) throw new Error('Invalid inventory cpuUsagePercent.'); + validateInstanceResources(inventory); for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`); if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.'); if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.'); @@ -253,6 +255,7 @@ export function enrollHost(token: unknown, rawInventory: unknown): { hostId: str revokeCredentials(enrollment.host_id); const credential = issueCredential(enrollment.host_id); database().prepare('UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), now(), inventory.hostname ?? '', JSON.stringify(inventory), enrollment.host_id); + recordResources(enrollment.host_id, inventory, now()); return { hostId: enrollment.host_id, credential, heartbeatSeconds: 15 }; })(); } @@ -284,9 +287,23 @@ export function rotateCredential(hostId: string, nextCredential?: string): strin } export function heartbeat(hostId: string, value: unknown): Host { requireHost(hostId); const inventory = validateInventory(value); - database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), inventory.hostname ?? '', JSON.stringify(inventory), hostId); + database().transaction(() => { + const timestamp = now(); + database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(timestamp, inventory.hostname ?? '', JSON.stringify(inventory), hostId); + recordResources(hostId, inventory, timestamp); + })(); return requireHost(hostId); } + +function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void { + recordInstanceResources(hostId, inventory, timestamp); + database().prepare(`INSERT OR REPLACE INTO ec_host_resource_samples + (host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,?,?,?,?)`) + .run(hostId, timestamp, inventory.cpuCount ?? null, inventory.cpuUsagePercent ?? null, + inventory.memoryTotal ?? null, inventory.memoryAvailable ?? null); + database().prepare('DELETE FROM ec_host_resource_samples WHERE host_id=? AND sampled_at 0) return previewNativeCreation(input); if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.'); let recipe: LaunchRecipe; let blueprintName: string | null = null; @@ -218,7 +219,7 @@ export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blue // releasingServers is internal to atomic collection queueing; HTTP callers cannot set it. export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult { const nativePreview = input.previewId ? db().prepare("SELECT 1 FROM server_launch_previews WHERE id=? AND json_extract(recipe,'$.native.hostId') IS NOT NULL").get(input.previewId) : null; - if (input.hostId || nativePreview || !config.komodo.address) return queueNativeCreation(input); + if (input.hostId !== undefined || nativePreview || !config.komodo.address || listHosts().length > 0) return queueNativeCreation(input); return db().transaction((): CreateServerResult => { const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined; if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' }; diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 976eadb..f626bba 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -1,9 +1,11 @@ +import { isIP } from 'node:net'; +import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js'; import { randomBytes, randomUUID } from 'node:crypto'; import { posix } from 'node:path'; import { db } from '../db/index.js'; import { config, relDestForKind, serverDir } from '../config.js'; import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js'; -import { getHost, getHostOperation, listHosts, nextHostPort, queueHostOperation, releaseHostPorts, requireReadyHost, reserveHostPort } from '../hosts/index.js'; +import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js'; import { newestReadyLibrary, requireReadyLibrary } from '../game-files/index.js'; import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js'; import { getServerKey, saveServerKey } from '../fleet/server-keys.js'; @@ -24,14 +26,14 @@ import type { ProvisioningAction } from '../../provisioning.js'; import { hostAgentUpdatesEnabled } from '../../hosts.js'; import { cancelCreation, finishCreationCancellation } from './cancel-creation.js'; -export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number } +export type NativeSelection = import('../../hosts.js').NativePlacement; type NativeRecipe = LaunchRecipe & { native?: NativeSelection }; interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string; instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null } interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null } -export function nativeSelection(input: Record, game: string): NativeSelection { - if (typeof input.hostId !== 'string' || !input.hostId) throw new Error('Select a ready host.'); +export function nativeSelection(input: Record, game: string, options: Pick = {}): NativeSelection { + if (input.hostId !== undefined && typeof input.hostId !== 'string') throw new Error('Invalid host.'); const version = newestReadyLibrary(game); if (!version) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`); if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== version.id) { @@ -39,14 +41,17 @@ export function nativeSelection(input: Record, game: string): N } const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image; if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.'); - const host = requireReadyHost(input.hostId, { image, requiredBytes: 1024 ** 3 }); - requireReadyLibrary(host.id, version.id); - const bindAddress = input.bindAddress === undefined || input.bindAddress === '' ? host.bindAddress : input.bindAddress; - if (typeof bindAddress !== 'string') throw new Error('Invalid bind address.'); - if (input.publicAddress !== undefined && (typeof input.publicAddress !== 'string' || input.publicAddress.length > 253)) throw new Error('Invalid public address.'); - if (input.publicPort !== undefined && (!Number.isInteger(input.publicPort) || Number(input.publicPort) < 1 || Number(input.publicPort) > 65535)) throw new Error('Invalid public port.'); - return { hostId: host.id, libraryId: version.id, image, bindAddress, publicAddress: input.publicAddress as string | undefined ?? host.publicAddress, - publicPort: input.publicPort as number | undefined }; + const automatic = input.automatic === true || !input.hostId; + const rawNetwork = input.automatic === true ? input.network ?? {} : input; + if (!rawNetwork || typeof rawNetwork !== 'object' || Array.isArray(rawNetwork)) throw new Error('Invalid network overrides.'); + const network = rawNetwork as NativeSelection; + const bindAddress = network.bindAddress || undefined; + if (bindAddress !== undefined && (typeof bindAddress !== 'string' || !isIP(bindAddress))) throw new Error('Invalid bind address.'); + const { host } = choosePlacement({ hostId: automatic ? undefined : input.hostId as string, libraryId: version.id, image, bindAddress, ...options }); + if (network.publicAddress !== undefined && (typeof network.publicAddress !== 'string' || network.publicAddress.length > 253)) throw new Error('Invalid public address.'); + if (network.publicPort !== undefined && (!Number.isInteger(network.publicPort) || Number(network.publicPort) < 1 || Number(network.publicPort) > 65535)) throw new Error('Invalid public port.'); + return { hostId: host.id, libraryId: version.id, image, bindAddress: bindAddress ?? host.bindAddress, publicAddress: network.publicAddress ?? host.publicAddress, + publicPort: network.publicPort, automatic, network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } }; } export function previewNativeCreation(input: Record): LaunchPreview { @@ -72,10 +77,10 @@ export function previewNativeCreation(input: Record): LaunchPre db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)') .run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName); const checks: LaunchPreview['checks'] = [ - { label: 'Host readiness', state: 'pass', detail: `${getHost(selection.hostId)!.name}: connected, Docker ready, storage and runtime verified.` }, - { label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on the selected host.` }, + { label: 'Host capacity', state: 'pass', detail: selection.automatic ? 'Capacity available. The least loaded eligible host is selected when provisioning starts.' : `${getHost(selection.hostId)!.name}: capacity available.` }, + { label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on an eligible host.` }, { label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' }, - { label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' }, + { label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? selection.automatic ? 'UDP/TCP capacity available; port assigned at placement.' : `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' }, { label: 'Configuration', state: 'pass', detail: recipe.configSource } ]; return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, checks, @@ -83,16 +88,9 @@ export function previewNativeCreation(input: Record): LaunchPre } export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null { - const excluded = [...taken]; - for (;;) { - const port = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'udp'); - if (port === null) return null; - // Both protocols are published by Plutainer and reserved together. - const tcp = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'tcp'); - if (tcp === port) return port; - if (tcp !== null && tcp < port) { excluded.push(tcp); continue; } - excluded.push(port); - } + const host = getHost(selection.hostId); + return host ? availableHostPort(host, { bindAddress: selection.bindAddress, + planned: taken.map(port => ({ hostId: host.id, port })) }) : null; } export function queueNativeCreation(input: CreateServerInput): CreateServerResult { @@ -113,7 +111,7 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul const name = `${game}_zombie_${input.mapSlug}_${lobby}`; const port = nextNativePort(selection); if (!port) throw new Error('No available host port.'); - if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || preview.port !== port))) throw new Error('Available lobby or host port changed. Review a fresh preview.'); + if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || (!selection.automatic && preview.port !== port)))) throw new Error('Available lobby or host port changed. Review a fresh preview.'); const key = input.plutoKey?.trim() || getServerKey(name); if (!key) throw new Error(`Save a Plutonium server key for ${name}.`); saveServerKey(name, key); @@ -128,11 +126,11 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)') .run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap); db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null); - if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=? WHERE id=?').run(id, preview.id); + if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id); deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' }); return { ok: true as const, name, port, deploymentId: id }; })(); - } catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; } + } catch (error) { return { ok: false, kind: error instanceof CapacityError ? 'conflict' : 'bad-request', message: (error as Error).message }; } } export function bindNativeOperation(id: string, server: string): void { @@ -270,12 +268,43 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s const activeJobs = new Map>(); +/** Reservations can move only while no remote work has ever been submitted. */ +function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance { + return db().transaction(() => { + const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe; + const prior = recipe.native!; + const network = prior.automatic ? prior.network ?? {} : prior; + const { host, port } = choosePlacement({ hostId: prior.automatic ? undefined : instance.host_id, + libraryId: instance.library_id, image: instance.image, bindAddress: network.bindAddress, + thorough: true, excludeInstanceId: instance.id }); + const bind = network.bindAddress ?? host.bindAddress; + releaseHostPorts(instance.id); + reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'udp' }); + reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'tcp' }); + db().prepare(`UPDATE native_instances SET host_id=?,bind_address=?,port=?,public_address=?,public_port=? WHERE id=?`) + .run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id); + db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id); + recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort }; + db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id); + db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id); + deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' }); + return nativeInstance(job.server)!; + })(); +} + async function processJob(job: Job, eligibility: (job: Job) => string | null) { if (!stillActive(job)) return; let instance = nativeInstance(job.server); if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) { finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return; } + if (job.action === 'create' && !job.operation_id && !job.stage_operation_id && !job.submitted_at) { + const settings = fleetSettings(); + const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL + AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number }; + if (active.n >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; } + instance = placeQueuedCreation(job, instance); + } const host = getHost(instance.host_id); if (!host?.connected || host.mode === 'disabled') { db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server); @@ -403,7 +432,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { } if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; } deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null }); - if (job.action === 'create') requireReadyHost(instance.host_id, { image: instance.image, requiredBytes: 1024 ** 3 }); + if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, + bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true }); if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id); if (job.action !== 'create' && job.action !== 'start') { // Fast controller check keeps occupied lobbies out of the operation slots. diff --git a/src/lib/ui/components/DeployCollectionDialog.svelte b/src/lib/ui/components/DeployCollectionDialog.svelte index bd01fea..9df8a30 100644 --- a/src/lib/ui/components/DeployCollectionDialog.svelte +++ b/src/lib/ui/components/DeployCollectionDialog.svelte @@ -17,7 +17,7 @@ let mode = $state('fill'); let games = $state([]); let collectionName = $state('Launch collection'); - let hostSelection = $state({ hostId: '', libraryIds: {}, image: '' }); + let hostSelection = $state({ libraryIds: {}, image: '' }); let hostReady = $state(false); let confirmRemovals = $state(false); let keys = $state>({}); @@ -41,7 +41,7 @@ async function initialize() { const current = ++session; busy = true; error = ''; games = []; hostReady = false; - hostSelection = { hostId: '', libraryIds: {}, image: '' }; + hostSelection = { libraryIds: {}, image: '' }; try { const response = await fetch('/api/blueprints'); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load collection.'); @@ -54,12 +54,12 @@ } async function review() { - if (!hostReady) { error = 'Choose a host and wait for its latest game files and runtime to finish automatic setup.'; return; } + if (!hostReady) { error = 'Wait for an eligible host and wait for its latest game files and runtime to finish automatic setup.'; return; } const current = ++session; const previous = preview; busy = true; error = ''; keyNotice = ''; confirmRemovals = false; try { - const res = await fetch('/api/deployment-collections/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ blueprintId, mode, hostId: hostSelection.hostId, libraryIds: hostSelection.libraryIds, image: hostSelection.image }) }); + const res = await fetch('/api/deployment-collections/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ blueprintId, mode, libraryIds: hostSelection.libraryIds, image: hostSelection.image }) }); const body = await res.json(); if (!res.ok) throw new Error(body.error ?? 'Could not preview the collection.'); if (current === session) { @@ -105,7 +105,7 @@ {#if open && games.length} { preview = null; confirmRemovals = false; }} />{/if}

{mode === 'exact' ? 'Keep matching lobbies, add missing ones, and remove excess lobbies and maps outside this collection across both games.' : 'Create missing lobbies and keep all existing servers.'}

{#if error}{/if} - {#if !preview}

{busy ? 'Preparing the collection…' : 'Choose a host. Its latest game files sync automatically before you review capacity and server keys.'}

{#if !busy}{/if} + {#if !preview}

{busy ? 'Preparing the collection…' : 'Wait for an eligible host. Its latest game files sync automatically before you review capacity and server keys.'}

{#if !busy}{/if} {:else} {#if keyNotice}

{keyNotice}

{/if} {#if expired}

This preview expired. Refresh before launch; keys remain entered for unchanged IDs.

{/if} @@ -133,7 +133,7 @@ {#each preview.servers as server, index (server.name ?? `unavailable-${index}`)}
  • -
    {#if index === 0 || preview.servers[index - 1].entryIndex !== server.entryIndex}{preview.entries[server.entryIndex].recipe.game.toUpperCase()} · {preview.entries[server.entryIndex].recipe.mapName}{/if}{#if server.name}{:else}{preview.entries[server.entryIndex].recipe.mapName}{/if}{server.name ?? 'No available lobby'} · {server.port === null ? 'No available port' : `Port ${server.port}`}
    +
    {#if index === 0 || preview.servers[index - 1].entryIndex !== server.entryIndex}{preview.entries[server.entryIndex].recipe.game.toUpperCase()} · {preview.entries[server.entryIndex].recipe.mapName}{/if}{#if server.name}{:else}{preview.entries[server.entryIndex].recipe.mapName}{/if}{server.name ?? 'No available lobby'} · {server.port === null ? 'No available capacity' : server.native?.automatic ? 'Host and port assigned automatically' : `Port ${server.port}`}
    {#if server.name} {#if server.hasSavedKey}

    Saved key ready

    {:else}{/if} @@ -145,7 +145,7 @@
    {#if preview.servers.length}

    {keysReady} / {preview.servers.length} keys available

    {/if}

    Key validity is checked at startup. Preview expires at {$timeFormat.minuteTime(preview.expiresAt)}.

    {#if keysReady < preview.servers.length}{/if}
    {:else}

    {preview.mode === 'exact' ? 'Your fleet already matches this collection.' : 'Your fleet already meets or exceeds every target in this collection.'} No servers will be created or removed.

    {/if} -
    Setup and launch requirements

    Each new server uses the selected host, latest game files, and prepared runtime. Host readiness, synced files, runtime availability, and local port reservations are checked again before deployment. Existing servers keep their current files.

    {#each preview.entries as entry}
    {/each}
    +
    Setup and launch requirements

    Each new server is automatically placed on an eligible host with the latest game files and prepared runtime. CPU, memory, host readiness, synced files, runtime availability, and local port reservations are checked again before deployment. Existing servers keep their current files.

    {#each preview.entries as entry}
    {/each}
    {/if} diff --git a/src/lib/ui/components/DeployServerDialog.svelte b/src/lib/ui/components/DeployServerDialog.svelte index a3a2f71..d0a84ba 100644 --- a/src/lib/ui/components/DeployServerDialog.svelte +++ b/src/lib/ui/components/DeployServerDialog.svelte @@ -23,7 +23,7 @@ let maps = $derived(catalogs[game]); let selectedBlueprint = $derived(blueprints.find(b => b.id === blueprintId)); let preview = $state(null); - let hostSelection = $state({ hostId: '', libraryIds: {}, image: '' }); + let hostSelection = $state({ libraryIds: {}, image: '' }); let hostReady = $state(false); let plutoKey = $state(''); let busy = $state(false); @@ -55,7 +55,7 @@ blueprintId = initialBlueprintId; game = initialGame; mapSlug = initialMapSlug; - hostSelection = { hostId: '', libraryIds: {}, image: '' }; hostReady = false; + hostSelection = { libraryIds: {}, image: '' }; hostReady = false; try { const [mapRes, blueprintRes] = await Promise.all([fetch('/api/maps'), fetch('/api/blueprints')]); if (!mapRes.ok || !blueprintRes.ok) throw new Error('Could not load maps and collections. Try again.'); @@ -76,7 +76,7 @@ } function selectionChanged() { - if (preview) selectionNotice = 'The host or game files changed. Review the launch again with the latest files.'; + if (preview) selectionNotice = 'The launch settings or game files changed. Review the launch again with the latest files.'; preview = null; plutoKey = ''; keyNotice = ''; } @@ -85,14 +85,14 @@ if (!sourceDeploymentId && selectedBlueprint) { open = false; collectionOpen = true; return; } - if (!hostReady) { error = 'Choose a host and wait for its latest game files and runtime to finish automatic setup.'; return; } + if (!hostReady) { error = 'Wait for a host to finish preparing the latest game files and runtime.'; return; } const current = session; const previousName = preview?.name; busy = true; error = ''; savedName = ''; keyNotice = ''; selectionNotice = ''; try { const res = await fetch('/api/provisioning/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ ...(sourceDeploymentId ? { deploymentId: sourceDeploymentId } : { game, mapSlug }), - hostId: hostSelection.hostId, libraryId: hostSelection.libraryIds[game], image: hostSelection.image, + libraryId: hostSelection.libraryIds[game], image: hostSelection.image, bindAddress: hostSelection.bindAddress || undefined, publicAddress: hostSelection.publicAddress || undefined, publicPort: hostSelection.publicPort }) }); const body = await res.json(); if (!res.ok) throw new Error(body.error ?? 'Could not prepare launch preview.'); @@ -154,13 +154,13 @@
      {#each preview.checks as check}
    • {check.state === 'pass' ? '✓' : check.state === 'blocked' ? '!' : 'i'}
      {check.label}

      {check.detail}

      • {/each}
    { event.preventDefault(); void launch(); }}> -
    Your new server{preview.name ?? 'Waiting for an available lobby'}

    {preview.port ? `Port ${preview.port} · ` : ''}Fresh identity, with its own server key.

    +
    Your new server{preview.name ?? 'Waiting for an available lobby'}

    Fresh identity, with its own server key.

    {#if preview.hasSavedKey}

    Saved key ready

    {:else}{/if}

    {preview.hasSavedKey ? 'The saved key for this server ID will be reused automatically. Electric Cherry does not check whether Plutonium accepts it. Update it in Keys.' : 'Add the key here. It is saved for this exact server ID when you launch, including if provisioning fails.'} Manage saved keys in a new tab; your launch stays open here.

    {#if !preview.hasSavedKey && !plutoKey.trim()}

    Enter this server’s key to enable launch.

    {/if} -

    Preview valid until {$timeFormat.minuteTime(preview.expiresAt)}. Lobby and port availability are checked again at launch.

    +

    Preview valid until {$timeFormat.minuteTime(preview.expiresAt)}. Capacity is checked again at launch. The host and port are assigned automatically.

    Make this a reusable collection

    Save this game and map with a target of one lobby. Future launches use the latest promoted setup and create only missing lobbies.

    {#if savedName}

    “{savedName}” is ready in your collection library.

    {:else} @@ -168,7 +168,7 @@ {/if}
    {:else if sourceDeploymentId} -

    Choose a host for the latest promoted {game.toUpperCase()} setup, with a fresh lobby and port.

    +

    Launch the latest promoted {game.toUpperCase()} setup, with a fresh lobby and port.

    {:else}
    { event.preventDefault(); void review(); }}> diff --git a/src/lib/ui/components/NativeHostSelection.svelte b/src/lib/ui/components/NativeHostSelection.svelte index 62f9e69..344e28a 100644 --- a/src/lib/ui/components/NativeHostSelection.svelte +++ b/src/lib/ui/components/NativeHostSelection.svelte @@ -1,7 +1,6 @@