Automatically place servers using learned resource capacity

This commit is contained in:
2026-10-04 21:04:09 -04:00
parent a1b957392f
commit e6cdf9eb7f
25 changed files with 927 additions and 85 deletions
+5 -3
View File
@@ -39,9 +39,11 @@ including SQLite metadata and published game-file storage.
shows setup progress. Automatic agent updates are enabled by default. shows setup progress. Automatic agent updates are enabled by default.
3. Add the exact Plutonium server keys in **Keys**, or supply missing keys when 3. Add the exact Plutonium server keys in **Keys**, or supply missing keys when
launching. launching.
4. Launch a server or collection on a host. The latest files for each game and 4. Launch a server or collection. EC quickly checks CPU, memory and port capacity,
the default runtime are selected automatically. EC reserves host-specific then automatically places each server on the least loaded eligible host. The
ports and verifies startup. latest game files and default runtime are selected automatically. EC reserves
resources for queued launches and verifies startup. CPU/memory allowances learn
from actual container usage, using temporary defaults while calibrating.
See [Native host management](docs/native-hosts.md) for installation, credentials, See [Native host management](docs/native-hosts.md) for installation, credentials,
storage, failure recovery, API contracts and the live-host acceptance checklist. storage, failure recovery, API contracts and the live-host acceptance checklist.
+60 -2
View File
@@ -9,6 +9,8 @@ import argparse
import base64 import base64
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
import contextlib import contextlib
from datetime import datetime, timezone
import math
import hashlib import hashlib
import fcntl import fcntl
import json import json
@@ -30,7 +32,7 @@ import urllib.parse
import urllib.request import urllib.request
import uuid import uuid
VERSION = "1.4.2" VERSION = "1.4.3"
LABEL = "io.electric-cherry" LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024 MAX_FILE = 32 * 1024 * 1024
LIBRARY_DOWNLOAD_WORKERS = 4 LIBRARY_DOWNLOAD_WORKERS = 4
@@ -929,6 +931,58 @@ class Agent:
return None return None
return round(100 * (total - deltas[3] - deltas[4]) / total, 1) return round(100 * (total - deltas[3] - deltas[4]) / total, 1)
def instance_resources(self):
"""One bounded Docker stats batch; unavailable samples are never zero load."""
ids = self.docker.call("container", "ls", "--filter", LABEL + ".managed=true",
"--no-trunc", "--format", "{{.ID}}", timeout=10).splitlines()[:1000]
if not ids:
return []
containers = json.loads(self.docker.call("container", "inspect", *ids, timeout=10))
owned = {}
for info in containers:
try:
labels = info.get("Config", {}).get("Labels", {}) or {}
if labels.get(LABEL + ".managed") != "true" or not info.get("State", {}).get("Running"):
continue
instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", "")))
name = self.container_name(instance_id, generation)
if info.get("Name", "").lstrip("/") != name:
continue
expected = str(self.instance_dir(instance_id, generation) / "app")
if not any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", [])):
continue
started = datetime.fromisoformat(info["State"]["StartedAt"].replace("Z", "+00:00"))
uptime = max(0, int((datetime.now(timezone.utc) - started).total_seconds()))
owned[name] = {"instanceId": instance_id, "generation": generation, "uptimeSeconds": uptime}
except (AgentError, ValueError, KeyError, TypeError):
continue
if not owned:
return []
rows = self.docker.call("stats", "--no-stream", "--no-trunc", "--format", "{{json .}}", *owned, timeout=15)
samples = []
units = {"B": 1, "KiB": 1024, "MiB": 1024 ** 2, "GiB": 1024 ** 3,
"TiB": 1024 ** 4, "kB": 1000, "MB": 1000 ** 2, "GB": 1000 ** 3}
for line in rows.splitlines():
try:
row = json.loads(line)
identity = owned.get(row.get("Name"))
if not identity or int(row["PIDs"]) <= 0:
continue
# Docker Linux CPU percent is core-based: 150% means 1.5 cores.
percent = row["CPUPerc"]
if not percent.endswith("%"):
continue
cores = float(percent[:-1]) / 100
used = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)", row["MemUsage"].split("/", 1)[0].strip())
if not used or used[2] not in units or not math.isfinite(cores) or cores < 0:
continue
memory = math.ceil(float(used[1]) * units[used[2]])
if memory > 0:
samples.append({**identity, "cpuCores": cores, "memoryBytes": memory})
except (ValueError, KeyError, TypeError, AttributeError, OverflowError):
continue
return samples
def inventory(self): def inventory(self):
cpu_usage = self.cpu_usage_percent() cpu_usage = self.cpu_usage_percent()
usage = shutil.disk_usage(self.root) usage = shutil.disk_usage(self.root)
@@ -1028,6 +1082,10 @@ class Agent:
if aliases.exists(): if aliases.exists():
with contextlib.suppress(OSError, ValueError): with contextlib.suppress(OSError, ValueError):
images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids) images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids)
resources = {}
if docker_running:
with contextlib.suppress(AgentError, ValueError, TypeError, KeyError):
resources["instanceResources"] = self.instance_resources()
os_name = platform.system() os_name = platform.system()
with contextlib.suppress(OSError): with contextlib.suppress(OSError):
for line in Path("/etc/os-release").read_text().splitlines(): for line in Path("/etc/os-release").read_text().splitlines():
@@ -1035,7 +1093,7 @@ class Agent:
os_name = line.split("=", 1)[1].strip('"') os_name = line.split("=", 1)[1].strip('"')
update = self.update_inventory() update = self.update_inventory()
capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
def safe_to_disrupt(self, instance_id, generation): def safe_to_disrupt(self, instance_id, generation):
health = self.inspect(instance_id, generation) health = self.inspect(instance_id, generation)
+44
View File
@@ -107,6 +107,50 @@ class AgentTests(unittest.TestCase):
else: else:
self.assertEqual(inventory["cpuUsagePercent"], usage) self.assertEqual(inventory["cpuUsagePercent"], usage)
def resource_container(self):
return {"Name": self.agent.container_name(INSTANCE, 1),
"Config": {"Labels": {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1"}},
"State": {"Running": True, "StartedAt": "2000-01-01T00:00:00.000000000Z"},
"Mounts": [{"Type": "bind", "Source": str(self.agent.instance_dir(INSTANCE, 1) / "app"), "Destination": "/home/plutainer/app"}]}
def test_instance_resources_batches_owned_containers_and_normalizes_core_cpu_and_memory(self):
name = self.agent.container_name(INSTANCE, 1)
rows = json.dumps({"Name": name, "CPUPerc": "150.00%", "MemUsage": "384.5MiB / 8GiB", "PIDs": "12"})
with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([self.resource_container()]), rows]) as call:
samples = self.agent.instance_resources()
self.assertEqual(len(samples), 1)
self.assertEqual(samples[0]["instanceId"], INSTANCE)
self.assertEqual(samples[0]["generation"], 1)
self.assertEqual(samples[0]["cpuCores"], 1.5)
self.assertEqual(samples[0]["memoryBytes"], int(384.5 * 1024 ** 2))
self.assertGreater(samples[0]["uptimeSeconds"], 300)
self.assertEqual(call.call_args_list[-1].args, ("stats", "--no-stream", "--no-trunc", "--format", "{{json .}}", name))
def test_instance_resources_ignores_stopped_foreign_and_mismatched_containers(self):
for change in ("stopped", "foreign", "generation", "name"):
info = self.resource_container()
if change == "stopped":
info["State"]["Running"] = False
elif change == "foreign":
info["Mounts"][0]["Source"] = "/another-installation/app"
elif change == "generation":
info["Config"]["Labels"][module.LABEL + ".generation"] = "2"
else:
info["Name"] = "unrelated"
with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([info])]) as call:
self.assertEqual(self.agent.instance_resources(), [])
self.assertEqual(call.call_count, 2)
def test_instance_resources_skips_unavailable_stats_instead_of_reporting_zero_usage(self):
good = {"Name": self.agent.container_name(INSTANCE, 1), "CPUPerc": "0.00%", "MemUsage": "200MiB / 8GiB", "PIDs": "10"}
for override in ({"CPUPerc": "NaN%"}, {"CPUPerc": "-1%"}, {"CPUPerc": "--"}, {"MemUsage": "bad"}, {"MemUsage": "0B / 0B"}, {"PIDs": "0"}, {"Name": "foreign"}):
with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([self.resource_container()]), json.dumps({**good, **override})]):
self.assertEqual(self.agent.instance_resources(), [])
with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([self.resource_container()]), json.dumps(good)]):
self.assertEqual(self.agent.instance_resources()[0]["cpuCores"], 0)
with patch.object(self.agent, "instance_resources", side_effect=AgentError("timed out")):
self.assertNotIn("instanceResources", self.agent.inventory())
def operation(self, kind, payload=None, op_id="operation-1", generation=1): def operation(self, kind, payload=None, op_id="operation-1", generation=1):
return {"id": op_id, "kind": kind, "instanceId": INSTANCE, "generation": generation, "payload": payload or {}} return {"id": op_id, "kind": kind, "instanceId": INSTANCE, "generation": generation, "payload": payload or {}}
+53 -3
View File
@@ -27,8 +27,8 @@ the agent owns local execution and its durable operation journal.
progress and errors. Wait for verification before launching. Hosts report progress and errors. Wait for verification before launching. Hosts report
connection and readiness separately; deployment also checks the particular connection and readiness separately; deployment also checks the particular
game files, image, capacity and ports required for the launch. game files, image, capacity and ports required for the launch.
6. Launch a server or collection. Select the host; the latest game files and 6. Launch a server or collection. The host, latest game files and default
default runtime are selected automatically. An advanced runtime override is runtime are selected automatically after checking capacity. An advanced runtime override is
available. If the latest files change after previewing, refresh the preview available. If the latest files change after previewing, refresh the preview
before launching. before launching.
EC selects lobby IDs and reserves UDP/TCP ports atomically. EC selects lobby IDs and reserves UDP/TCP ports atomically.
@@ -121,7 +121,57 @@ host-wide busy time across all cores on a 0–100% scale between inventory sampl
(normally about 15 seconds). Idle and I/O wait are excluded; stolen CPU time (normally about 15 seconds). Idle and I/O wait are excluded; stolen CPU time
counts as unavailable capacity. The first sample after startup, unavailable counts as unavailable capacity. The first sample after startup, unavailable
counters, and older agents show **Not reported**. Offline hosts retain their counters, and older agents show **Not reported**. Offline hosts retain their
last reported value. This is current inventory, not historical metrics. last reported value. The controller retains CPU and memory samples for seven days (missing measurements
remain unknown). Scheduling uses the last five minutes as well as the latest
heartbeat; the Inventory panel continues to show the latest reading.
New servers are placed automatically. Preview and submission perform local
capacity checks without contacting agents: a connected, active host must have
verified game files, its runtime, storage, a free UDP/TCP port, and valid CPU and
memory measurements. Hosts with stale or missing measurements cannot accept new
servers. Existing API clients may still explicitly constrain a launch to a host.
Admission reserves a provisional host and port atomically, including collection
launches. Agent **1.4.3** samples each managed running container with a bounded
[`docker stats`](https://docs.docker.com/reference/cli/docker/container/stats/)
batch during inventory collection. CPU is normalized to cores (100% Docker CPU
is one core); memory is the Linux working usage reported by Docker, excluding
reclaimable cache. Missing or failed samples stay unknown instead of becoming
zero usage. Samples must match the host's live instance and generation.
The controller retains per-instance minute peaks for 24 hours and caches learned
CPU/memory allowances. The first five minutes after a container starts count as
startup; later samples train the running allowance. Startup uses the observed
maximum and running uses the 95th percentile of minute peaks, both with **25%
headroom**. Each calculation considers up to the latest 10,000 minute peaks.
Host-local evidence is preferred to account for different hardware; new hosts
can borrow the fleet's learned allowance. Startup allowances are never lower
than running allowances. The host page shows both allowances and their source.
Lower allowances require at least three startup minute samples spanning two
minutes, or twenty running minute samples spanning five minutes. Until then,
**0.5 core and 1 GiB** are temporary cold-start allowances, raised immediately
if observations exceed them. They are replaced by measured allowances once the
evidence threshold is met; a tiny 0.01-core floor prevents zero-CPU admission.
Models expire after 24 hours without measurements. Recalculation happens during
heartbeat ingestion, at most once per minute per host/fleet phase; new peaks
raise the cache immediately. Launch admission reads only the cached model.
Scheduling still leaves **10% host CPU and 512 MiB RAM** for the OS and bursts.
These allowances are scheduling reservations, not Docker limits. Queued and
unresolved creations consume the startup allowance; established instances keep
the running allowance even while idle or stopped. Observed host load also counts,
so other processes and previously unseen load increases reduce available capacity.
When startup capacity is available, provisioning rechecks all eligible hosts and
chooses the lowest projected CPU/memory utilization, using the busier of the
latest reading and the five-minute average. It moves the provisional reservation
atomically and records the final host in the launch recipe. Host network defaults
are resolved again for the final host; explicit overrides are preserved. If no
host qualifies, the request waits and is checked again. Once remote staging or
creation has been submitted, placement stays fixed to prevent duplicate instances.
Cancelling an unsubmitted launch releases its reservation; uncertain remote work
retains capacity until cleanup is confirmed.
When publishing changed agent code, bump its numeric `VERSION` in When publishing changed agent code, bump its numeric `VERSION` in
`agent/ec_agent.py` and rebuild the controller. Packaging derives the manifest `agent/ec_agent.py` and rebuild the controller. Packaging derives the manifest
+1 -1
View File
@@ -45,7 +45,7 @@ export interface CollectionPreview {
removals: { name: string; reason: string }[]; removals: { name: string; reason: string }[];
entries: { count: number; existing: number; missing: number; existingServers: string[]; entries: { count: number; existing: number; missing: number; existingServers: string[];
coverage: { name: string; health: string; href: string }[]; recipe: LaunchRecipeSummary }[]; coverage: { name: string; health: string; href: string }[]; recipe: LaunchRecipeSummary }[];
servers: { entryIndex: number; name: string | null; port: number | null; hasSavedKey: boolean }[]; servers: { native?: import('./hosts.js').NativePlacement; entryIndex: number; name: string | null; port: number | null; hasSavedKey: boolean }[];
blockers: string[]; blockers: string[];
ready: boolean; ready: boolean;
} }
+18 -1
View File
@@ -68,7 +68,16 @@ export interface HostInventory {
currentOperation?: string | null; [key: string]: unknown; currentOperation?: string | null; [key: string]: unknown;
capabilities?: string[]; capabilities?: string[];
agentUpdate?: HostAgentUpdate; agentUpdate?: HostAgentUpdate;
instanceResources?: InstanceResources[];
} }
export interface InstanceResources {
instanceId: string; generation: number; uptimeSeconds: number; cpuCores: number; memoryBytes: number;
}
export interface ResourceAllowance {
cpuCores: number; memoryBytes: number; source: 'host' | 'fleet' | 'bootstrap';
samples: number; observedAt: string | null;
}
export interface CapacityCalibration { running: ResourceAllowance; startup: ResourceAllowance }
export interface Host { export interface Host {
id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number; id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number;
bindAddress: string; publicAddress: string; minimumFreeBytes: number; createdAt: string; bindAddress: string; publicAddress: string; minimumFreeBytes: number; createdAt: string;
@@ -95,6 +104,14 @@ export interface HostPreparation {
} }
export interface NativeDeploymentSelection { export interface NativeDeploymentSelection {
hostId: string; libraryIds: Partial<Record<'t5' | 't6', string>>; image: string; libraryIds: Partial<Record<'t5' | 't6', string>>; image: string;
bindAddress?: string; publicAddress?: string; publicPort?: number; bindAddress?: string; publicAddress?: string; publicPort?: number;
} }
/** The host/port are provisional until the creation command is submitted. */
export interface NativePlacement {
hostId: string; libraryId: string; image?: string;
bindAddress?: string; publicAddress?: string; publicPort?: number;
automatic?: boolean;
network?: { bindAddress?: string; publicAddress?: string; publicPort?: number };
}
+2
View File
@@ -9,6 +9,7 @@ export function migrateNativeInstances(handle: Database.Database): void {
port INTEGER NOT NULL, public_address TEXT, public_port INTEGER, port INTEGER NOT NULL, public_address TEXT, public_port INTEGER,
created_at TEXT NOT NULL, retired_at TEXT created_at TEXT NOT NULL, retired_at TEXT
); );
CREATE INDEX IF NOT EXISTS native_instance_host ON native_instances(host_id) WHERE retired_at IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS native_instance_live_name ON native_instances(server) WHERE retired_at IS NULL; CREATE UNIQUE INDEX IF NOT EXISTS native_instance_live_name ON native_instances(server) WHERE retired_at IS NULL;
CREATE TABLE IF NOT EXISTS native_provisioning ( CREATE TABLE IF NOT EXISTS native_provisioning (
deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id), deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
@@ -16,6 +17,7 @@ export function migrateNativeInstances(handle: Database.Database): void {
stage_operation_id TEXT, operation_id TEXT, files_json TEXT, stage_operation_id TEXT, operation_id TEXT, files_json TEXT,
applied_at TEXT, verified_at TEXT applied_at TEXT, verified_at TEXT
); );
CREATE INDEX IF NOT EXISTS native_provisioning_instance ON native_provisioning(instance_id);
CREATE TABLE IF NOT EXISTS native_creation_cancellations ( CREATE TABLE IF NOT EXISTS native_creation_cancellations (
creation_id TEXT PRIMARY KEY REFERENCES server_deployments(id), creation_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
cleanup_id TEXT NOT NULL UNIQUE REFERENCES server_deployments(id) cleanup_id TEXT NOT NULL UNIQUE REFERENCES server_deployments(id)
+16 -6
View File
@@ -12,6 +12,7 @@ import { serverObservation } from './status.js';
import { activeBetaRemovalBlocker, getServer } from './servers.js'; import { activeBetaRemovalBlocker, getServer } from './servers.js';
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js'; import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
import { nativeSelection, nextNativePort } from '../provisioning/native.js'; import { nativeSelection, nextNativePort } from '../provisioning/native.js';
import { listHosts } from '../hosts/index.js';
import { nativeInstance } from '../backends/native.js'; import { nativeInstance } from '../backends/native.js';
function coverage(entries: BlueprintEntry[]) { function coverage(entries: BlueprintEntry[]) {
@@ -73,15 +74,24 @@ function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode) {
function allocate(entries: BlueprintEntry[], releasingServers: string[] = []): CollectionPreview['servers'] { function allocate(entries: BlueprintEntry[], releasingServers: string[] = []): CollectionPreview['servers'] {
const names: string[] = []; const names: string[] = [];
const ports: number[] = []; const ports: number[] = [];
const planned: { hostId: string; port: number }[] = [];
const counts = coverage(entries); const counts = coverage(entries);
return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => { return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`; const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
const lobby = nextLobbyNumber(base, names); const lobby = nextLobbyNumber(base, names);
const name = lobby === null ? null : `${base}_${lobby}`; const name = lobby === null ? null : `${base}_${lobby}`;
const port = entry.recipe.native ? nextNativePort(entry.recipe.native, ports) : nextAvailablePort(ports, releasingServers); let native = entry.recipe.native;
let port: number | null;
if (native) {
try {
native = nativeSelection(native as unknown as Record<string, unknown>, entry.recipe.game, { planned });
port = nextNativePort(native, planned.filter(item => item.hostId === native!.hostId).map(item => item.port));
if (port !== null) planned.push({ hostId: native.hostId, port });
} catch { port = null; }
} else port = nextAvailablePort(ports, releasingServers);
if (name) names.push(name); if (name) names.push(name);
if (port !== null) ports.push(port); if (port !== null) ports.push(port);
return { entryIndex, name, port, hasSavedKey: !!name && !!getServerKey(name) }; return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && !!getServerKey(name) };
})); }));
} }
@@ -98,7 +108,7 @@ function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEn
if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.'); if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.');
} }
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.'); if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
if (servers.some(server => server.port === null)) blockers.push(`There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`); if (servers.some(server => server.port === null)) blockers.push(entries.some(entry => entry.recipe.native) ? 'There is not enough host CPU, memory or port capacity for the whole collection.' : `There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
return blockers; return blockers;
} }
@@ -117,7 +127,7 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill',
if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.'); if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.');
return db().transaction((): CollectionPreview => { return db().transaction((): CollectionPreview => {
const { entries, blueprintName } = blueprintEntries(blueprintId); const { entries, blueprintName } = blueprintEntries(blueprintId);
if (selection?.hostId !== undefined || !config.komodo.address) { if (selection?.hostId !== undefined || !config.komodo.address || listHosts().length > 0) {
const libraries = selection?.libraryIds as Record<string, unknown> | undefined; const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
for (const entry of entries) { for (const entry of entries) {
entry.recipe.native = nativeSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game); entry.recipe.native = nativeSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
@@ -159,7 +169,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
const servers = allocate(entries, releasingServers); const servers = allocate(entries, releasingServers);
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)]; const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
if (blockers.length) throw new Error(blockers.join(' ')); if (blockers.length) throw new Error(blockers.join(' '));
if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || server.port !== reviewed[i]?.port)) { if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || (!server.native?.automatic && server.port !== reviewed[i]?.port))) {
throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.'); throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.');
} }
const expected = new Set(servers.map(server => server.name)); const expected = new Set(servers.map(server => server.name));
@@ -196,7 +206,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
return { id, ports }; return { id, ports };
}); });
servers.forEach((server, index) => { servers.forEach((server, index) => {
const recipe = entries[server.entryIndex].recipe; const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) };
const childPreviewId = randomUUID(); const childPreviewId = randomUUID();
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)') db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
.run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name); .run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name);
+1 -1
View File
@@ -9,7 +9,7 @@ import { getBaselineForScope, getVersion, listPromotedOverlaysForScope } from '.
import { sha256Buffer } from './files.js'; import { sha256Buffer } from './files.js';
export interface LaunchRecipe { export interface LaunchRecipe {
native?: { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }; native?: import('../../hosts.js').NativePlacement;
format: 1; format: 1;
game: ZombieGame; game: ZombieGame;
mapSlug: string; mapSlug: string;
+106
View File
@@ -0,0 +1,106 @@
import { db } from '../db/index.js';
import { ensureHostSchema } from './schema.js';
import type { CapacityCalibration, HostInventory, ResourceAllowance } from '../../hosts.js';
const BOOTSTRAP_CPU = 0.5;
const BOOTSTRAP_MEMORY = 1024 ** 3;
const MARGIN = 1.25;
const WINDOW_MS = 24 * 60 * 60_000;
const MAX_SAMPLES = 10_000;
type Phase = 'startup' | 'running';
interface AllowanceRow {
scope: string; phase: Phase; cpu_cores: number; memory_bytes: number; samples: number;
window_start: string; observed_at: string; computed_at: string;
}
function database() { const handle = db(); ensureHostSchema(handle); return handle; }
export function validateInstanceResources(inventory: HostInventory): void {
const samples = inventory.instanceResources;
if (samples === undefined) return;
if (!Array.isArray(samples) || samples.length > 1000) throw new Error('Invalid instance resource samples.');
const seen = new Set<string>();
for (const sample of samples) {
if (!sample || typeof sample !== 'object' || typeof sample.instanceId !== 'string' || !sample.instanceId || sample.instanceId.length > 100
|| !Number.isSafeInteger(sample.generation) || sample.generation < 1
|| !Number.isSafeInteger(sample.uptimeSeconds) || sample.uptimeSeconds < 0
|| typeof sample.cpuCores !== 'number' || !Number.isFinite(sample.cpuCores) || sample.cpuCores < 0
|| !Number.isSafeInteger(sample.memoryBytes) || sample.memoryBytes <= 0
|| (inventory.cpuCount !== undefined && sample.cpuCores > inventory.cpuCount + 0.01)
|| (inventory.memoryTotal !== undefined && sample.memoryBytes > inventory.memoryTotal)
|| seen.has(sample.instanceId)) throw new Error('Invalid instance resource sample.');
seen.add(sample.instanceId);
}
}
/** Heartbeat ingestion owns all learning work. Launch requests only read cached allowances. */
export function recordInstanceResources(hostId: string, inventory: HostInventory, timestamp: string): void {
if (!inventory.instanceResources?.length || !inventory.dockerRunning || !inventory.cpuCount || !inventory.memoryTotal) return;
const handle = database();
const bucket = timestamp.slice(0, 16) + ':00.000Z';
const observed = new Map<Phase, { cpu: number; memory: number }>();
const owned = handle.prepare('SELECT 1 FROM native_instances WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL');
const insert = handle.prepare(`INSERT INTO ec_instance_resource_samples
(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,?,?,?,?,?,?)
ON CONFLICT(host_id,instance_id,generation,phase,bucket) DO UPDATE SET
cpu_cores=MAX(cpu_cores,excluded.cpu_cores), memory_bytes=MAX(memory_bytes,excluded.memory_bytes)`);
for (const sample of inventory.instanceResources) {
// An agent can only train from its currently owned generation; foreign,
// retired and delayed pre-rebuild samples cannot contaminate the model.
if (!owned.get(sample.instanceId, hostId, sample.generation)) continue;
const phase: Phase = sample.uptimeSeconds < 300 ? 'startup' : 'running';
insert.run(hostId, sample.instanceId, sample.generation, phase, bucket, sample.cpuCores, sample.memoryBytes);
const prior = observed.get(phase);
observed.set(phase, { cpu: Math.max(prior?.cpu ?? 0, sample.cpuCores), memory: Math.max(prior?.memory ?? 0, sample.memoryBytes) });
}
for (const [phase, peak] of observed) for (const scope of [hostId, '*']) {
const previous = handle.prepare('SELECT * FROM ec_resource_allowances WHERE scope=? AND phase=?').get(scope, phase) as AllowanceRow | undefined;
if (!previous || previous.computed_at < bucket) {
const rows = handle.prepare(`SELECT cpu_cores, memory_bytes, bucket FROM ec_instance_resource_samples
WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`)
.all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as
{ cpu_cores: number; memory_bytes: number; bucket: string }[];
const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1];
const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu) * MARGIN;
const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory) * MARGIN);
handle.prepare(`INSERT INTO ec_resource_allowances
(scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(scope,phase) DO UPDATE SET
cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples,
window_start=excluded.window_start,observed_at=excluded.observed_at,computed_at=excluded.computed_at`)
.run(scope, phase, cpu, memory, rows.length, rows.at(-1)!.bucket, timestamp, timestamp);
if (scope === '*') handle.prepare('DELETE FROM ec_instance_resource_samples WHERE bucket<?')
.run(new Date(Date.parse(timestamp) - WINDOW_MS).toISOString());
} else {
// Raise the allowance immediately on a new peak; a lower sample
// cannot reduce it until the next scheduled percentile calculation.
handle.prepare(`UPDATE ec_resource_allowances SET cpu_cores=MAX(cpu_cores,?),memory_bytes=MAX(memory_bytes,?),observed_at=? WHERE scope=? AND phase=?`)
.run(peak.cpu * MARGIN, Math.ceil(peak.memory * MARGIN), timestamp, scope, phase);
}
}
}
/** Host measurements account for hardware differences; fleet evidence covers new hosts. */
export function capacityCalibration(hostId: string): CapacityCalibration {
const rows = database().prepare('SELECT * FROM ec_resource_allowances WHERE scope IN (?,?) AND observed_at>=?')
.all(hostId, '*', new Date(Date.now() - WINDOW_MS).toISOString()) as AllowanceRow[];
const allowance = (phase: Phase): ResourceAllowance => {
const local = rows.find(row => row.scope === hostId && row.phase === phase);
const fleet = rows.find(row => row.scope === '*' && row.phase === phase);
const learned = (row?: AllowanceRow) => !!row && row.samples >= (phase === 'startup' ? 3 : 20)
&& Date.parse(row.observed_at) - Date.parse(row.window_start) >= (phase === 'startup' ? 120_000 : 300_000);
const selected = learned(local) ? local : learned(fleet) ? fleet : undefined;
return {
// A few higher observations can increase a fleet/bootstrap allowance
// immediately. Decreasing it requires the evidence thresholds above.
cpuCores: Math.max(0.01, selected?.cpu_cores ?? BOOTSTRAP_CPU, local?.cpu_cores ?? 0, selected ? 0 : fleet?.cpu_cores ?? 0),
memoryBytes: Math.max(selected?.memory_bytes ?? BOOTSTRAP_MEMORY, local?.memory_bytes ?? 0, selected ? 0 : fleet?.memory_bytes ?? 0),
source: selected ? selected.scope === hostId ? 'host' : 'fleet' : 'bootstrap',
samples: selected?.samples ?? local?.samples ?? fleet?.samples ?? 0,
observedAt: selected?.observed_at ?? local?.observed_at ?? fleet?.observed_at ?? null
};
};
const running = allowance('running'); const startup = allowance('startup');
if (running.source === 'bootstrap' && (running.cpuCores > startup.cpuCores || running.memoryBytes > startup.memoryBytes)) startup.source = 'bootstrap';
startup.cpuCores = Math.max(startup.cpuCores, running.cpuCores);
startup.memoryBytes = Math.max(startup.memoryBytes, running.memoryBytes);
return { running, startup };
}
+83
View File
@@ -0,0 +1,83 @@
import { db } from '../db/index.js';
import { listHosts, requireReadyHost } from './index.js';
import { requireReadyLibrary } from '../game-files/index.js';
import type { Host } from '../../hosts.js';
import { capacityCalibration } from './calibration.js';
// Storage and OS headroom are separate from learned instance CPU/memory usage.
const INSTANCE_STORAGE_BYTES = 1024 ** 3;
const MEMORY_HEADROOM = 512 * 1024 ** 2;
export class CapacityError extends Error {}
export interface PlacementOptions {
hostId?: string; libraryId: string; image: string; bindAddress?: string;
thorough?: boolean; excludeInstanceId?: string;
planned?: { hostId: string; port: number }[];
}
/** One indexed port read per host, including both protocols and wildcard binds. */
export function availableHostPort(host: Host, options: Pick<PlacementOptions, 'bindAddress' | 'excludeInstanceId' | 'planned'> = {}): number | null {
const bind = options.bindAddress ?? host.bindAddress;
const rows = db().prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?')
.all(host.id, options.excludeInstanceId ?? '') as { port: number; bind_address: string }[];
const used = new Set(rows.filter(row => row.bind_address === bind || ['0.0.0.0', '::'].includes(bind)
|| ['0.0.0.0', '::'].includes(row.bind_address)).map(row => row.port));
for (const item of options.planned ?? []) if (item.hostId === host.id) used.add(item.port);
for (let port = host.portMin; port <= host.portMax; port++) if (!used.has(port)) return port;
return null;
}
function projectedLoad(host: Host, options: PlacementOptions): number {
const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
if (!cpuCount || cpuUsagePercent === undefined || !memoryTotal || memoryAvailable === undefined || memoryAvailable > memoryTotal) {
throw new Error('Waiting for valid CPU and memory measurements.');
}
let cpu = cpuUsagePercent;
let available = memoryAvailable;
if (options.thorough) {
const history = db().prepare(`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory
FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`)
.get(host.id, new Date(Date.now() - 5 * 60_000).toISOString(), cpuCount, memoryTotal) as { cpu: number | null; memory: number | null };
cpu = Math.max(cpu, history.cpu ?? cpu);
available = Math.min(available, history.memory ?? available);
}
// Unfinished/uncertain launches remain charged until recovery and a later
// heartbeat. Observed usage accounts for established servers; the count
// floor also reserves their minimum footprint while they are idle/stopped.
const instances = db().prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(NOT EXISTS (
SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE n.instance_id=i.id AND d.action='create' AND d.state IN ('done','warning') AND d.finished_at<?
)),0) AS pending FROM native_instances i WHERE i.host_id=? AND i.retired_at IS NULL AND i.id!=?`)
.get(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as { total: number; pending: number };
const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
const count = instances.total + planned + 1;
const pending = instances.pending + planned + 1;
const { running, startup } = capacityCalibration(host.id);
const cores = Math.max(cpuCount * cpu / 100 + pending * startup.cpuCores,
(count - pending) * running.cpuCores + pending * startup.cpuCores);
const memory = Math.max(memoryTotal - available + pending * startup.memoryBytes,
(count - pending) * running.memoryBytes + pending * startup.memoryBytes);
if (cores > cpuCount * 0.9) throw new Error('Insufficient CPU capacity.');
if (memory > memoryTotal - MEMORY_HEADROOM) throw new Error('Insufficient memory capacity.');
return Math.max(cores / cpuCount, memory / memoryTotal);
}
/** Admission stops at the first fit; placement evaluates and ranks every fit. No remote I/O. */
export function choosePlacement(options: PlacementOptions): { host: Host; port: number; load: number } {
const candidates: { host: Host; port: number; load: number }[] = [];
const reasons: string[] = [];
for (const host of listHosts().filter(host => !options.hostId || host.id === options.hostId)) {
try {
requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES });
requireReadyLibrary(host.id, options.libraryId);
const port = availableHostPort(host, options);
if (port === null) throw new Error('No available UDP/TCP game port.');
const load = projectedLoad(host, options);
const candidate = { host, port, load };
if (!options.thorough) return candidate;
candidates.push(candidate);
} catch (error) { reasons.push(`${host.name}: ${(error as Error).message}`); }
}
candidates.sort((a, b) => a.load - b.load || a.host.id.localeCompare(b.host.id));
if (candidates.length) return candidates[0];
throw new CapacityError(`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`);
}
+18 -1
View File
@@ -4,6 +4,7 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypt
import { isIP } from 'node:net'; import { isIP } from 'node:net';
import { db } from '../db/index.js'; import { db } from '../db/index.js';
import { ensureHostSchema } from './schema.js'; import { ensureHostSchema } from './schema.js';
import { recordInstanceResources, validateInstanceResources } from './calibration.js';
import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js'; import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
export const AGENT_PROTOCOL_VERSION = 1; export const AGENT_PROTOCOL_VERSION = 1;
@@ -219,6 +220,7 @@ export function validateInventory(value: unknown): HostInventory {
if (inventory[key] !== undefined && (typeof inventory[key] !== 'number' || !Number.isSafeInteger(inventory[key]) || Number(inventory[key]) < 0)) throw new Error(`Invalid inventory ${key}.`); if (inventory[key] !== undefined && (typeof inventory[key] !== 'number' || !Number.isSafeInteger(inventory[key]) || Number(inventory[key]) < 0)) throw new Error(`Invalid inventory ${key}.`);
} }
if (inventory.cpuUsagePercent !== undefined && (typeof inventory.cpuUsagePercent !== 'number' || !Number.isFinite(inventory.cpuUsagePercent) || inventory.cpuUsagePercent < 0 || inventory.cpuUsagePercent > 100)) throw new Error('Invalid inventory cpuUsagePercent.'); if (inventory.cpuUsagePercent !== undefined && (typeof inventory.cpuUsagePercent !== 'number' || !Number.isFinite(inventory.cpuUsagePercent) || inventory.cpuUsagePercent < 0 || inventory.cpuUsagePercent > 100)) throw new Error('Invalid inventory cpuUsagePercent.');
validateInstanceResources(inventory);
for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`); for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.'); if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.'); if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
@@ -253,6 +255,7 @@ export function enrollHost(token: unknown, rawInventory: unknown): { hostId: str
revokeCredentials(enrollment.host_id); revokeCredentials(enrollment.host_id);
const credential = issueCredential(enrollment.host_id); const credential = issueCredential(enrollment.host_id);
database().prepare('UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), now(), inventory.hostname ?? '', JSON.stringify(inventory), enrollment.host_id); database().prepare('UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), now(), inventory.hostname ?? '', JSON.stringify(inventory), enrollment.host_id);
recordResources(enrollment.host_id, inventory, now());
return { hostId: enrollment.host_id, credential, heartbeatSeconds: 15 }; return { hostId: enrollment.host_id, credential, heartbeatSeconds: 15 };
})(); })();
} }
@@ -284,9 +287,23 @@ export function rotateCredential(hostId: string, nextCredential?: string): strin
} }
export function heartbeat(hostId: string, value: unknown): Host { export function heartbeat(hostId: string, value: unknown): Host {
requireHost(hostId); const inventory = validateInventory(value); requireHost(hostId); const inventory = validateInventory(value);
database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), inventory.hostname ?? '', JSON.stringify(inventory), hostId); database().transaction(() => {
const timestamp = now();
database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(timestamp, inventory.hostname ?? '', JSON.stringify(inventory), hostId);
recordResources(hostId, inventory, timestamp);
})();
return requireHost(hostId); return requireHost(hostId);
} }
function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void {
recordInstanceResources(hostId, inventory, timestamp);
database().prepare(`INSERT OR REPLACE INTO ec_host_resource_samples
(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,?,?,?,?)`)
.run(hostId, timestamp, inventory.cpuCount ?? null, inventory.cpuUsagePercent ?? null,
inventory.memoryTotal ?? null, inventory.memoryAvailable ?? null);
database().prepare('DELETE FROM ec_host_resource_samples WHERE host_id=? AND sampled_at<?')
.run(hostId, new Date(Date.parse(timestamp) - 7 * 86400_000).toISOString());
}
export function requireReadyHost(hostId: string, options: { libraryId?: string; requiredBytes?: number; image?: string; port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): Host { export function requireReadyHost(hostId: string, options: { libraryId?: string; requiredBytes?: number; image?: string; port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): Host {
const host = requireHost(hostId); const reasons = [...host.readinessReasons]; const host = requireHost(hostId); const reasons = [...host.readinessReasons];
if (options.requiredBytes !== undefined && (host.inventory.storageAvailable ?? 0) < options.requiredBytes + host.minimumFreeBytes) reasons.push('Insufficient storage for this deployment.'); if (options.requiredBytes !== undefined && (host.inventory.storageAvailable ?? 0) < options.requiredBytes + host.minimumFreeBytes) reasons.push('Insufficient storage for this deployment.');
+19
View File
@@ -40,6 +40,25 @@ export function ensureHostSchema(handle: Database.Database): void {
PRIMARY KEY(host_id,bind_address,protocol,port) PRIMARY KEY(host_id,bind_address,protocol,port)
); );
CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id); CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id);
CREATE TABLE IF NOT EXISTS ec_host_resource_samples (
host_id TEXT NOT NULL REFERENCES ec_hosts(id), sampled_at TEXT NOT NULL,
cpu_count REAL, cpu_usage_percent REAL, memory_total REAL, memory_available REAL,
PRIMARY KEY(host_id, sampled_at)
);
CREATE TABLE IF NOT EXISTS ec_instance_resource_samples (
host_id TEXT NOT NULL, instance_id TEXT NOT NULL, generation INTEGER NOT NULL,
phase TEXT NOT NULL CHECK(phase IN ('startup','running')), bucket TEXT NOT NULL,
cpu_cores REAL NOT NULL, memory_bytes REAL NOT NULL,
PRIMARY KEY(host_id,instance_id,generation,phase,bucket)
);
CREATE INDEX IF NOT EXISTS ec_instance_resources_host ON ec_instance_resource_samples(host_id,phase,bucket DESC);
CREATE INDEX IF NOT EXISTS ec_instance_resources_fleet ON ec_instance_resource_samples(phase,bucket DESC);
CREATE INDEX IF NOT EXISTS ec_instance_resources_retention ON ec_instance_resource_samples(bucket);
CREATE TABLE IF NOT EXISTS ec_resource_allowances (
scope TEXT NOT NULL, phase TEXT NOT NULL, cpu_cores REAL NOT NULL, memory_bytes REAL NOT NULL,
samples INTEGER NOT NULL, window_start TEXT NOT NULL, observed_at TEXT NOT NULL, computed_at TEXT NOT NULL,
PRIMARY KEY(scope,phase)
);
CREATE TABLE IF NOT EXISTS ec_host_decommissions ( CREATE TABLE IF NOT EXISTS ec_host_decommissions (
host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id), host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id),
operation_id TEXT NOT NULL UNIQUE REFERENCES ec_host_operations(id), operation_id TEXT NOT NULL UNIQUE REFERENCES ec_host_operations(id),
+3 -2
View File
@@ -1,3 +1,4 @@
import { listHosts } from '../hosts/index.js';
import { fleetSettings } from '../settings.js'; import { fleetSettings } from '../settings.js';
import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence.js'; import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence.js';
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js'; import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
@@ -176,7 +177,7 @@ interface PreviewRow {
/** Read-only checks; a preview never reserves a slot or writes credentials. */ /** Read-only checks; a preview never reserves a slot or writes credentials. */
export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown; hostId?: unknown; libraryId?: unknown }): LaunchPreview { export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown; hostId?: unknown; libraryId?: unknown }): LaunchPreview {
if (input.hostId !== undefined || !config.komodo.address) return previewNativeCreation(input); if (input.hostId !== undefined || !config.komodo.address || listHosts().length > 0) return previewNativeCreation(input);
if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.'); if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.');
let recipe: LaunchRecipe; let recipe: LaunchRecipe;
let blueprintName: string | null = null; let blueprintName: string | null = null;
@@ -218,7 +219,7 @@ export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blue
// releasingServers is internal to atomic collection queueing; HTTP callers cannot set it. // releasingServers is internal to atomic collection queueing; HTTP callers cannot set it.
export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult { export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult {
const nativePreview = input.previewId ? db().prepare("SELECT 1 FROM server_launch_previews WHERE id=? AND json_extract(recipe,'$.native.hostId') IS NOT NULL").get(input.previewId) : null; const nativePreview = input.previewId ? db().prepare("SELECT 1 FROM server_launch_previews WHERE id=? AND json_extract(recipe,'$.native.hostId') IS NOT NULL").get(input.previewId) : null;
if (input.hostId || nativePreview || !config.komodo.address) return queueNativeCreation(input); if (input.hostId !== undefined || nativePreview || !config.komodo.address || listHosts().length > 0) return queueNativeCreation(input);
return db().transaction((): CreateServerResult => { return db().transaction((): CreateServerResult => {
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined; const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined;
if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' }; if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' };
+59 -29
View File
@@ -1,9 +1,11 @@
import { isIP } from 'node:net';
import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
import { randomBytes, randomUUID } from 'node:crypto'; import { randomBytes, randomUUID } from 'node:crypto';
import { posix } from 'node:path'; import { posix } from 'node:path';
import { db } from '../db/index.js'; import { db } from '../db/index.js';
import { config, relDestForKind, serverDir } from '../config.js'; import { config, relDestForKind, serverDir } from '../config.js';
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js'; import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
import { getHost, getHostOperation, listHosts, nextHostPort, queueHostOperation, releaseHostPorts, requireReadyHost, reserveHostPort } from '../hosts/index.js'; import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
import { newestReadyLibrary, requireReadyLibrary } from '../game-files/index.js'; import { newestReadyLibrary, requireReadyLibrary } from '../game-files/index.js';
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js'; import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
import { getServerKey, saveServerKey } from '../fleet/server-keys.js'; import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
@@ -24,14 +26,14 @@ import type { ProvisioningAction } from '../../provisioning.js';
import { hostAgentUpdatesEnabled } from '../../hosts.js'; import { hostAgentUpdatesEnabled } from '../../hosts.js';
import { cancelCreation, finishCreationCancellation } from './cancel-creation.js'; import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number } export type NativeSelection = import('../../hosts.js').NativePlacement;
type NativeRecipe = LaunchRecipe & { native?: NativeSelection }; type NativeRecipe = LaunchRecipe & { native?: NativeSelection };
interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string; interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string;
instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null } instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null }
interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null } interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null }
export function nativeSelection(input: Record<string, unknown>, game: string): NativeSelection { export function nativeSelection(input: Record<string, unknown>, game: string, options: Pick<PlacementOptions, 'planned'> = {}): NativeSelection {
if (typeof input.hostId !== 'string' || !input.hostId) throw new Error('Select a ready host.'); if (input.hostId !== undefined && typeof input.hostId !== 'string') throw new Error('Invalid host.');
const version = newestReadyLibrary(game); const version = newestReadyLibrary(game);
if (!version) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`); if (!version) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`);
if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== version.id) { if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== version.id) {
@@ -39,14 +41,17 @@ export function nativeSelection(input: Record<string, unknown>, game: string): N
} }
const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image; const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.'); if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
const host = requireReadyHost(input.hostId, { image, requiredBytes: 1024 ** 3 }); const automatic = input.automatic === true || !input.hostId;
requireReadyLibrary(host.id, version.id); const rawNetwork = input.automatic === true ? input.network ?? {} : input;
const bindAddress = input.bindAddress === undefined || input.bindAddress === '' ? host.bindAddress : input.bindAddress; if (!rawNetwork || typeof rawNetwork !== 'object' || Array.isArray(rawNetwork)) throw new Error('Invalid network overrides.');
if (typeof bindAddress !== 'string') throw new Error('Invalid bind address.'); const network = rawNetwork as NativeSelection;
if (input.publicAddress !== undefined && (typeof input.publicAddress !== 'string' || input.publicAddress.length > 253)) throw new Error('Invalid public address.'); const bindAddress = network.bindAddress || undefined;
if (input.publicPort !== undefined && (!Number.isInteger(input.publicPort) || Number(input.publicPort) < 1 || Number(input.publicPort) > 65535)) throw new Error('Invalid public port.'); if (bindAddress !== undefined && (typeof bindAddress !== 'string' || !isIP(bindAddress))) throw new Error('Invalid bind address.');
return { hostId: host.id, libraryId: version.id, image, bindAddress, publicAddress: input.publicAddress as string | undefined ?? host.publicAddress, const { host } = choosePlacement({ hostId: automatic ? undefined : input.hostId as string, libraryId: version.id, image, bindAddress, ...options });
publicPort: input.publicPort as number | undefined }; if (network.publicAddress !== undefined && (typeof network.publicAddress !== 'string' || network.publicAddress.length > 253)) throw new Error('Invalid public address.');
if (network.publicPort !== undefined && (!Number.isInteger(network.publicPort) || Number(network.publicPort) < 1 || Number(network.publicPort) > 65535)) throw new Error('Invalid public port.');
return { hostId: host.id, libraryId: version.id, image, bindAddress: bindAddress ?? host.bindAddress, publicAddress: network.publicAddress ?? host.publicAddress,
publicPort: network.publicPort, automatic, network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } };
} }
export function previewNativeCreation(input: Record<string, unknown>): LaunchPreview { export function previewNativeCreation(input: Record<string, unknown>): LaunchPreview {
@@ -72,10 +77,10 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)') db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)')
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName); .run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
const checks: LaunchPreview['checks'] = [ const checks: LaunchPreview['checks'] = [
{ label: 'Host readiness', state: 'pass', detail: `${getHost(selection.hostId)!.name}: connected, Docker ready, storage and runtime verified.` }, { label: 'Host capacity', state: 'pass', detail: selection.automatic ? 'Capacity available. The least loaded eligible host is selected when provisioning starts.' : `${getHost(selection.hostId)!.name}: capacity available.` },
{ label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on the selected host.` }, { label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on an eligible host.` },
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' }, { label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' },
{ label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' }, { label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? selection.automatic ? 'UDP/TCP capacity available; port assigned at placement.' : `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
{ label: 'Configuration', state: 'pass', detail: recipe.configSource } { label: 'Configuration', state: 'pass', detail: recipe.configSource }
]; ];
return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, checks, return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, checks,
@@ -83,16 +88,9 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
} }
export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null { export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null {
const excluded = [...taken]; const host = getHost(selection.hostId);
for (;;) { return host ? availableHostPort(host, { bindAddress: selection.bindAddress,
const port = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'udp'); planned: taken.map(port => ({ hostId: host.id, port })) }) : null;
if (port === null) return null;
// Both protocols are published by Plutainer and reserved together.
const tcp = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'tcp');
if (tcp === port) return port;
if (tcp !== null && tcp < port) { excluded.push(tcp); continue; }
excluded.push(port);
}
} }
export function queueNativeCreation(input: CreateServerInput): CreateServerResult { export function queueNativeCreation(input: CreateServerInput): CreateServerResult {
@@ -113,7 +111,7 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
const name = `${game}_zombie_${input.mapSlug}_${lobby}`; const name = `${game}_zombie_${input.mapSlug}_${lobby}`;
const port = nextNativePort(selection); const port = nextNativePort(selection);
if (!port) throw new Error('No available host port.'); if (!port) throw new Error('No available host port.');
if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || preview.port !== port))) throw new Error('Available lobby or host port changed. Review a fresh preview.'); if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || (!selection.automatic && preview.port !== port)))) throw new Error('Available lobby or host port changed. Review a fresh preview.');
const key = input.plutoKey?.trim() || getServerKey(name); const key = input.plutoKey?.trim() || getServerKey(name);
if (!key) throw new Error(`Save a Plutonium server key for ${name}.`); if (!key) throw new Error(`Save a Plutonium server key for ${name}.`);
saveServerKey(name, key); saveServerKey(name, key);
@@ -128,11 +126,11 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)') db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
.run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap); .run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null); db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null);
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=? WHERE id=?').run(id, preview.id); if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id);
deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' }); deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' });
return { ok: true as const, name, port, deploymentId: id }; return { ok: true as const, name, port, deploymentId: id };
})(); })();
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; } } catch (error) { return { ok: false, kind: error instanceof CapacityError ? 'conflict' : 'bad-request', message: (error as Error).message }; }
} }
export function bindNativeOperation(id: string, server: string): void { export function bindNativeOperation(id: string, server: string): void {
@@ -270,12 +268,43 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
const activeJobs = new Map<string, Promise<void>>(); const activeJobs = new Map<string, Promise<void>>();
/** Reservations can move only while no remote work has ever been submitted. */
function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance {
return db().transaction(() => {
const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe;
const prior = recipe.native!;
const network = prior.automatic ? prior.network ?? {} : prior;
const { host, port } = choosePlacement({ hostId: prior.automatic ? undefined : instance.host_id,
libraryId: instance.library_id, image: instance.image, bindAddress: network.bindAddress,
thorough: true, excludeInstanceId: instance.id });
const bind = network.bindAddress ?? host.bindAddress;
releaseHostPorts(instance.id);
reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'udp' });
reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'tcp' });
db().prepare(`UPDATE native_instances SET host_id=?,bind_address=?,port=?,public_address=?,public_port=? WHERE id=?`)
.run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id);
db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id);
recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort };
db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id);
deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' });
return nativeInstance(job.server)!;
})();
}
async function processJob(job: Job, eligibility: (job: Job) => string | null) { async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (!stillActive(job)) return; if (!stillActive(job)) return;
let instance = nativeInstance(job.server); let instance = nativeInstance(job.server);
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) { if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return; finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
} }
if (job.action === 'create' && !job.operation_id && !job.stage_operation_id && !job.submitted_at) {
const settings = fleetSettings();
const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
if (active.n >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
instance = placeQueuedCreation(job, instance);
}
const host = getHost(instance.host_id); const host = getHost(instance.host_id);
if (!host?.connected || host.mode === 'disabled') { if (!host?.connected || host.mode === 'disabled') {
db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server); db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server);
@@ -403,7 +432,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
} }
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; } if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null }); deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
if (job.action === 'create') requireReadyHost(instance.host_id, { image: instance.image, requiredBytes: 1024 ** 3 }); if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true });
if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id); if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id);
if (job.action !== 'create' && job.action !== 'start') { if (job.action !== 'create' && job.action !== 'start') {
// Fast controller check keeps occupied lobbies out of the operation slots. // Fast controller check keeps occupied lobbies out of the operation slots.
@@ -17,7 +17,7 @@
let mode = $state<CollectionMode>('fill'); let mode = $state<CollectionMode>('fill');
let games = $state<ZombieGame[]>([]); let games = $state<ZombieGame[]>([]);
let collectionName = $state('Launch collection'); let collectionName = $state('Launch collection');
let hostSelection = $state<NativeDeploymentSelection>({ hostId: '', libraryIds: {}, image: '' }); let hostSelection = $state<NativeDeploymentSelection>({ libraryIds: {}, image: '' });
let hostReady = $state(false); let hostReady = $state(false);
let confirmRemovals = $state(false); let confirmRemovals = $state(false);
let keys = $state<Record<string, string>>({}); let keys = $state<Record<string, string>>({});
@@ -41,7 +41,7 @@
async function initialize() { async function initialize() {
const current = ++session; busy = true; error = ''; games = []; hostReady = false; const current = ++session; busy = true; error = ''; games = []; hostReady = false;
hostSelection = { hostId: '', libraryIds: {}, image: '' }; hostSelection = { libraryIds: {}, image: '' };
try { try {
const response = await fetch('/api/blueprints'); const body = await response.json(); const response = await fetch('/api/blueprints'); const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not load collection.'); if (!response.ok) throw new Error(body.error ?? 'Could not load collection.');
@@ -54,12 +54,12 @@
} }
async function review() { async function review() {
if (!hostReady) { error = 'Choose a host and wait for its latest game files and runtime to finish automatic setup.'; return; } if (!hostReady) { error = 'Wait for an eligible host and wait for its latest game files and runtime to finish automatic setup.'; return; }
const current = ++session; const current = ++session;
const previous = preview; const previous = preview;
busy = true; error = ''; keyNotice = ''; confirmRemovals = false; busy = true; error = ''; keyNotice = ''; confirmRemovals = false;
try { try {
const res = await fetch('/api/deployment-collections/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ blueprintId, mode, hostId: hostSelection.hostId, libraryIds: hostSelection.libraryIds, image: hostSelection.image }) }); const res = await fetch('/api/deployment-collections/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ blueprintId, mode, libraryIds: hostSelection.libraryIds, image: hostSelection.image }) });
const body = await res.json(); const body = await res.json();
if (!res.ok) throw new Error(body.error ?? 'Could not preview the collection.'); if (!res.ok) throw new Error(body.error ?? 'Could not preview the collection.');
if (current === session) { if (current === session) {
@@ -105,7 +105,7 @@
{#if open && games.length}<NativeHostSelection {games} bind:selection={hostSelection} bind:ready={hostReady} disabled={busy} onchange={() => { preview = null; confirmRemovals = false; }} />{/if} {#if open && games.length}<NativeHostSelection {games} bind:selection={hostSelection} bind:ready={hostReady} disabled={busy} onchange={() => { preview = null; confirmRemovals = false; }} />{/if}
<p class="muted">{mode === 'exact' ? 'Keep matching lobbies, add missing ones, and remove excess lobbies and maps outside this collection across both games.' : 'Create missing lobbies and keep all existing servers.'}</p> <p class="muted">{mode === 'exact' ? 'Keep matching lobbies, add missing ones, and remove excess lobbies and maps outside this collection across both games.' : 'Create missing lobbies and keep all existing servers.'}</p>
{#if error}<p class="error" role="alert">{error}</p>{/if} {#if error}<p class="error" role="alert">{error}</p>{/if}
{#if !preview}<p class="muted">{busy ? 'Preparing the collection…' : 'Choose a host. Its latest game files sync automatically before you review capacity and server keys.'}</p>{#if !busy}<Button disabled={!hostReady} onclick={review}>Preview launch</Button>{/if} {#if !preview}<p class="muted">{busy ? 'Preparing the collection…' : 'Wait for an eligible host. Its latest game files sync automatically before you review capacity and server keys.'}</p>{#if !busy}<Button disabled={!hostReady} onclick={review}>Preview launch</Button>{/if}
{:else} {:else}
{#if keyNotice}<p role="status" class="muted">{keyNotice}</p>{/if} {#if keyNotice}<p role="status" class="muted">{keyNotice}</p>{/if}
{#if expired}<p class="error" role="status">This preview expired. Refresh before launch; keys remain entered for unchanged IDs.</p>{/if} {#if expired}<p class="error" role="status">This preview expired. Refresh before launch; keys remain entered for unchanged IDs.</p>{/if}
@@ -133,7 +133,7 @@
{#each preview.servers as server, index (server.name ?? `unavailable-${index}`)} {#each preview.servers as server, index (server.name ?? `unavailable-${index}`)}
<li class="server-row"> <li class="server-row">
<span class="order" aria-hidden="true">{String(index + 1).padStart(2, '0')}</span> <span class="order" aria-hidden="true">{String(index + 1).padStart(2, '0')}</span>
<div class="server-identity">{#if index === 0 || preview.servers[index - 1].entryIndex !== server.entryIndex}<span class="map-group">{preview.entries[server.entryIndex].recipe.game.toUpperCase()} · {preview.entries[server.entryIndex].recipe.mapName}</span>{/if}<strong>{#if server.name}<ServerIdentity name={server.name} />{:else}{preview.entries[server.entryIndex].recipe.mapName}{/if}</strong><span class="server-id">{server.name ?? 'No available lobby'} · {server.port === null ? 'No available port' : `Port ${server.port}`}</span></div> <div class="server-identity">{#if index === 0 || preview.servers[index - 1].entryIndex !== server.entryIndex}<span class="map-group">{preview.entries[server.entryIndex].recipe.game.toUpperCase()} · {preview.entries[server.entryIndex].recipe.mapName}</span>{/if}<strong>{#if server.name}<ServerIdentity name={server.name} />{:else}{preview.entries[server.entryIndex].recipe.mapName}{/if}</strong><span class="server-id">{server.name ?? 'No available lobby'} · {server.port === null ? 'No available capacity' : server.native?.automatic ? 'Host and port assigned automatically' : `Port ${server.port}`}</span></div>
{#if server.name} {#if server.name}
{#if server.hasSavedKey}<p class="key-ready"><span aria-hidden="true">✓</span> Saved key ready</p> {#if server.hasSavedKey}<p class="key-ready"><span aria-hidden="true">✓</span> Saved key ready</p>
{:else}<label><span>Server key required</span><input aria-label={`Key for ${server.name}`} data-missing={!keys[server.name]?.trim()} type="password" bind:value={keys[server.name]} disabled={busy} maxlength="512" autocomplete="off" placeholder="Paste this server’s key" /></label>{/if} {:else}<label><span>Server key required</span><input aria-label={`Key for ${server.name}`} data-missing={!keys[server.name]?.trim()} type="password" bind:value={keys[server.name]} disabled={busy} maxlength="512" autocomplete="off" placeholder="Paste this server’s key" /></label>{/if}
@@ -145,7 +145,7 @@
<div class="launch-footer"><div>{#if preview.servers.length}<p role="status"><strong>{keysReady} / {preview.servers.length} keys available</strong></p>{/if}<p class="caption">Key validity is checked at startup. Preview expires at {$timeFormat.minuteTime(preview.expiresAt)}.</p>{#if keysReady < preview.servers.length}<button class="text-button" type="button" onclick={nextMissing}>Jump to next missing key</button>{/if}</div><div class="actions"><Button variant="ghost" disabled={busy} onclick={() => open = false}>Cancel</Button><Button type="submit" variant={preview.removals.length ? 'danger' : 'primary'} loading={busy} disabled={busy || expired || !canLaunch}>{preview.mode === 'exact' ? `Apply: ${preview.servers.length} create, ${preview.removals.length} remove` : `Queue ${preview.servers.length} server${preview.servers.length === 1 ? '' : 's'} →`}</Button></div></div> <div class="launch-footer"><div>{#if preview.servers.length}<p role="status"><strong>{keysReady} / {preview.servers.length} keys available</strong></p>{/if}<p class="caption">Key validity is checked at startup. Preview expires at {$timeFormat.minuteTime(preview.expiresAt)}.</p>{#if keysReady < preview.servers.length}<button class="text-button" type="button" onclick={nextMissing}>Jump to next missing key</button>{/if}</div><div class="actions"><Button variant="ghost" disabled={busy} onclick={() => open = false}>Cancel</Button><Button type="submit" variant={preview.removals.length ? 'danger' : 'primary'} loading={busy} disabled={busy || expired || !canLaunch}>{preview.mode === 'exact' ? `Apply: ${preview.servers.length} create, ${preview.removals.length} remove` : `Queue ${preview.servers.length} server${preview.servers.length === 1 ? '' : 's'} →`}</Button></div></div>
</form> </form>
{:else}<p class="muted">{preview.mode === 'exact' ? 'Your fleet already matches this collection.' : 'Your fleet already meets or exceeds every target in this collection.'} No servers will be created or removed.</p><div class="actions"><Button onclick={() => open = false}>Done</Button></div>{/if} {:else}<p class="muted">{preview.mode === 'exact' ? 'Your fleet already matches this collection.' : 'Your fleet already meets or exceeds every target in this collection.'} No servers will be created or removed.</p><div class="actions"><Button onclick={() => open = false}>Done</Button></div>{/if}
<details class="recipes"><summary>Setup and launch requirements</summary><p class="muted requirements">Each new server uses the selected host, latest game files, and prepared runtime. Host readiness, synced files, runtime availability, and local port reservations are checked again before deployment. Existing servers keep their current files.</p>{#each preview.entries as entry}<div class="recipe"><LaunchRecipe recipe={entry.recipe} /></div>{/each}</details> <details class="recipes"><summary>Setup and launch requirements</summary><p class="muted requirements">Each new server is automatically placed on an eligible host with the latest game files and prepared runtime. CPU, memory, host readiness, synced files, runtime availability, and local port reservations are checked again before deployment. Existing servers keep their current files.</p>{#each preview.entries as entry}<div class="recipe"><LaunchRecipe recipe={entry.recipe} /></div>{/each}</details>
{/if} {/if}
</div> </div>
</dialog> </dialog>
@@ -23,7 +23,7 @@
let maps = $derived(catalogs[game]); let maps = $derived(catalogs[game]);
let selectedBlueprint = $derived(blueprints.find(b => b.id === blueprintId)); let selectedBlueprint = $derived(blueprints.find(b => b.id === blueprintId));
let preview = $state<LaunchPreview | null>(null); let preview = $state<LaunchPreview | null>(null);
let hostSelection = $state<NativeDeploymentSelection>({ hostId: '', libraryIds: {}, image: '' }); let hostSelection = $state<NativeDeploymentSelection>({ libraryIds: {}, image: '' });
let hostReady = $state(false); let hostReady = $state(false);
let plutoKey = $state(''); let plutoKey = $state('');
let busy = $state(false); let busy = $state(false);
@@ -55,7 +55,7 @@
blueprintId = initialBlueprintId; blueprintId = initialBlueprintId;
game = initialGame; game = initialGame;
mapSlug = initialMapSlug; mapSlug = initialMapSlug;
hostSelection = { hostId: '', libraryIds: {}, image: '' }; hostReady = false; hostSelection = { libraryIds: {}, image: '' }; hostReady = false;
try { try {
const [mapRes, blueprintRes] = await Promise.all([fetch('/api/maps'), fetch('/api/blueprints')]); const [mapRes, blueprintRes] = await Promise.all([fetch('/api/maps'), fetch('/api/blueprints')]);
if (!mapRes.ok || !blueprintRes.ok) throw new Error('Could not load maps and collections. Try again.'); if (!mapRes.ok || !blueprintRes.ok) throw new Error('Could not load maps and collections. Try again.');
@@ -76,7 +76,7 @@
} }
function selectionChanged() { function selectionChanged() {
if (preview) selectionNotice = 'The host or game files changed. Review the launch again with the latest files.'; if (preview) selectionNotice = 'The launch settings or game files changed. Review the launch again with the latest files.';
preview = null; plutoKey = ''; keyNotice = ''; preview = null; plutoKey = ''; keyNotice = '';
} }
@@ -85,14 +85,14 @@
if (!sourceDeploymentId && selectedBlueprint) { if (!sourceDeploymentId && selectedBlueprint) {
open = false; collectionOpen = true; return; open = false; collectionOpen = true; return;
} }
if (!hostReady) { error = 'Choose a host and wait for its latest game files and runtime to finish automatic setup.'; return; } if (!hostReady) { error = 'Wait for a host to finish preparing the latest game files and runtime.'; return; }
const current = session; const current = session;
const previousName = preview?.name; const previousName = preview?.name;
busy = true; error = ''; savedName = ''; keyNotice = ''; selectionNotice = ''; busy = true; error = ''; savedName = ''; keyNotice = ''; selectionNotice = '';
try { try {
const res = await fetch('/api/provisioning/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, const res = await fetch('/api/provisioning/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ...(sourceDeploymentId ? { deploymentId: sourceDeploymentId } : { game, mapSlug }), body: JSON.stringify({ ...(sourceDeploymentId ? { deploymentId: sourceDeploymentId } : { game, mapSlug }),
hostId: hostSelection.hostId, libraryId: hostSelection.libraryIds[game], image: hostSelection.image, libraryId: hostSelection.libraryIds[game], image: hostSelection.image,
bindAddress: hostSelection.bindAddress || undefined, publicAddress: hostSelection.publicAddress || undefined, publicPort: hostSelection.publicPort }) }); bindAddress: hostSelection.bindAddress || undefined, publicAddress: hostSelection.publicAddress || undefined, publicPort: hostSelection.publicPort }) });
const body = await res.json(); const body = await res.json();
if (!res.ok) throw new Error(body.error ?? 'Could not prepare launch preview.'); if (!res.ok) throw new Error(body.error ?? 'Could not prepare launch preview.');
@@ -154,13 +154,13 @@
<ul>{#each preview.checks as check}<li class:blocked={check.state === 'blocked'}><span class="check" aria-label={check.state}>{check.state === 'pass' ? '✓' : check.state === 'blocked' ? '!' : 'i'}</span><div><strong>{check.label}</strong><p>{check.detail}</p></div></li>{/each}</ul> <ul>{#each preview.checks as check}<li class:blocked={check.state === 'blocked'}><span class="check" aria-label={check.state}>{check.state === 'pass' ? '✓' : check.state === 'blocked' ? '!' : 'i'}</span><div><strong>{check.label}</strong><p>{check.detail}</p></div></li>{/each}</ul>
</section> </section>
<form onsubmit={event => { event.preventDefault(); void launch(); }}> <form onsubmit={event => { event.preventDefault(); void launch(); }}>
<div class="identity"><span class="eyebrow">Your new server</span><strong>{preview.name ?? 'Waiting for an available lobby'}</strong><p>{preview.port ? `Port ${preview.port} · ` : ''}Fresh identity, with its own server key.</p></div> <div class="identity"><span class="eyebrow">Your new server</span><strong>{preview.name ?? 'Waiting for an available lobby'}</strong><p>Fresh identity, with its own server key.</p></div>
{#if preview.hasSavedKey}<p class="key-ready"><span aria-hidden="true">✓</span> Saved key ready</p> {#if preview.hasSavedKey}<p class="key-ready"><span aria-hidden="true">✓</span> Saved key ready</p>
{:else}<label>Plutonium server key<input type="password" bind:value={plutoKey} disabled={busy || saving || !preview.name} maxlength="512" autocomplete="off" placeholder="Paste the key for this server ID" /></label>{/if} {:else}<label>Plutonium server key<input type="password" bind:value={plutoKey} disabled={busy || saving || !preview.name} maxlength="512" autocomplete="off" placeholder="Paste the key for this server ID" /></label>{/if}
<p class="muted">{preview.hasSavedKey ? 'The saved key for this server ID will be reused automatically. Electric Cherry does not check whether Plutonium accepts it. Update it in Keys.' : 'Add the key here. It is saved for this exact server ID when you launch, including if provisioning fails.'} <a href={`/server-keys?server=${encodeURIComponent(preview.name ?? '')}`} target="_blank" rel="noopener">Manage saved keys in a new tab</a>; your launch stays open here.</p> <p class="muted">{preview.hasSavedKey ? 'The saved key for this server ID will be reused automatically. Electric Cherry does not check whether Plutonium accepts it. Update it in Keys.' : 'Add the key here. It is saved for this exact server ID when you launch, including if provisioning fails.'} <a href={`/server-keys?server=${encodeURIComponent(preview.name ?? '')}`} target="_blank" rel="noopener">Manage saved keys in a new tab</a>; your launch stays open here.</p>
<div class="launch-actions"><Button variant="ghost" disabled={busy || saving} onclick={() => { preview = null; plutoKey = ''; error = ''; }}>Back</Button><Button type="submit" variant="primary" loading={busy} disabled={busy || saving || expired || !hostReady || !preview.ready || (!preview.hasSavedKey && !plutoKey.trim())}>Launch server</Button></div> <div class="launch-actions"><Button variant="ghost" disabled={busy || saving} onclick={() => { preview = null; plutoKey = ''; error = ''; }}>Back</Button><Button type="submit" variant="primary" loading={busy} disabled={busy || saving || expired || !hostReady || !preview.ready || (!preview.hasSavedKey && !plutoKey.trim())}>Launch server</Button></div>
{#if !preview.hasSavedKey && !plutoKey.trim()}<p class="muted">Enter this server’s key to enable launch.</p>{/if} {#if !preview.hasSavedKey && !plutoKey.trim()}<p class="muted">Enter this server’s key to enable launch.</p>{/if}
<p class="caption">Preview valid until {$timeFormat.minuteTime(preview.expiresAt)}. Lobby and port availability are checked again at launch.</p> <p class="caption">Preview valid until {$timeFormat.minuteTime(preview.expiresAt)}. Capacity is checked again at launch. The host and port are assigned automatically.</p>
</form> </form>
<details class="save"><summary>Make this a reusable collection</summary><p class="muted">Save this game and map with a target of one lobby. Future launches use the latest promoted setup and create only missing lobbies.</p> <details class="save"><summary>Make this a reusable collection</summary><p class="muted">Save this game and map with a target of one lobby. Future launches use the latest promoted setup and create only missing lobbies.</p>
{#if savedName}<p class="success" role="status">“{savedName}” is ready in your collection library.</p>{:else} {#if savedName}<p class="success" role="status">“{savedName}” is ready in your collection library.</p>{:else}
@@ -168,7 +168,7 @@
{/if} {/if}
</details> </details>
{:else if sourceDeploymentId} {:else if sourceDeploymentId}
<p class="muted">Choose a host for the latest promoted {game.toUpperCase()} setup, with a fresh lobby and port.</p><Button disabled={busy || !hostReady} onclick={review}>Review launch</Button> <p class="muted">Launch the latest promoted {game.toUpperCase()} setup, with a fresh lobby and port.</p><Button disabled={busy || !hostReady} onclick={review}>Review launch</Button>
{:else} {:else}
<form onsubmit={event => { event.preventDefault(); void review(); }}> <form onsubmit={event => { event.preventDefault(); void review(); }}>
<label>Start with<select bind:value={blueprintId} disabled={busy}><option value="">Current promoted setup</option>{#each blueprints as blueprint}<option value={blueprint.id}>{blueprint.name} · {blueprint.totalServers} server{blueprint.totalServers === 1 ? '' : 's'}</option>{/each}</select></label> <label>Start with<select bind:value={blueprintId} disabled={busy}><option value="">Current promoted setup</option>{#each blueprints as blueprint}<option value={blueprint.id}>{blueprint.name} · {blueprint.totalServers} server{blueprint.totalServers === 1 ? '' : 's'}</option>{/each}</select></label>
@@ -1,7 +1,6 @@
<script lang="ts"> <script lang="ts">
import { onMount } from 'svelte'; import { onMount } from 'svelte';
import type { Host, NativeDeploymentSelection } from '$lib/hosts.js'; import type { Host, NativeDeploymentSelection } from '$lib/hosts.js';
import { hostReadinessLabel } from '$lib/hosts.js';
import type { GameLibrary } from '$lib/game-files.js'; import type { GameLibrary } from '$lib/game-files.js';
import type { ZombieGame } from '$lib/map-aliases.js'; import type { ZombieGame } from '$lib/map-aliases.js';
import { pollVisible } from '$ui/poll.js'; import { pollVisible } from '$ui/poll.js';
@@ -12,11 +11,11 @@
let hosts = $state<Host[]>([]); let libraries = $state<GameLibrary[]>([]); let loading = $state(true); let error = $state(''); let hosts = $state<Host[]>([]); let libraries = $state<GameLibrary[]>([]); let loading = $state(true); let error = $state('');
let defaultRuntimeImage = $state(''); let defaultRuntimeImage = $state('');
let runtimeSelection: NativeDeploymentSelection | null = null; let runtimeSelection: NativeDeploymentSelection | null = null;
const host = $derived(hosts.find(item => item.id === selection.hostId)); const eligible = $derived(hosts.filter(host => host.readiness === 'ready' && host.inventory.runtimeImages?.includes(selection.image)));
const latest = $derived(latestGameFiles(libraries)); const latest = $derived(latestGameFiles(libraries));
const imageReady = $derived(!!host?.inventory.runtimeImages?.includes(selection.image));
function cacheReady(game: ZombieGame) { return !!latest[game]?.caches.some(cache => cache.hostId === selection.hostId && ['ready', 'outdated'].includes(cache.status)); } function cacheReady(game: ZombieGame) { return !!latest[game]?.caches.some(cache => eligible.some(host => host.id === cache.hostId) && ['ready', 'outdated'].includes(cache.status)); }
$effect(() => { ready = !loading && !error && host?.readiness === 'ready' && imageReady && games.length > 0 && games.every(game => selection.libraryIds[game] === latest[game]?.id && cacheReady(game)); }); $effect(() => { ready = !loading && !error && games.length > 0 && games.every(game => selection.libraryIds[game] === latest[game]?.id && cacheReady(game)); });
$effect(() => { $effect(() => {
if (disabled || !defaultRuntimeImage || runtimeSelection === selection) return; if (disabled || !defaultRuntimeImage || runtimeSelection === selection) return;
if (!selection.image) selection.image = defaultRuntimeImage; if (!selection.image) selection.image = defaultRuntimeImage;
@@ -43,16 +42,14 @@
<div class="title"><h3>Host and game files</h3><button type="button" disabled={disabled || loading} onclick={() => { onchange?.(); void load(); }}>Refresh hosts</button></div> <div class="title"><h3>Host and game files</h3><button type="button" disabled={disabled || loading} onclick={() => { onchange?.(); void load(); }}>Refresh hosts</button></div>
{#if error}<p role="alert" class="error">{error}</p>{/if} {#if error}<p role="alert" class="error">{error}</p>{/if}
{#if loading}<p role="status">Loading host readiness and game files…</p>{/if} {#if loading}<p role="status">Loading host readiness and game files…</p>{/if}
<label>Game host<select bind:value={selection.hostId} disabled={disabled || loading} onchange={() => onchange?.()}><option value="">Choose a host</option>{#each hosts as candidate}<option value={candidate.id} disabled={candidate.mode !== 'active' || !!candidate.decommission || !candidate.credentialActive}>{candidate.name} · {candidate.connectionState} · {candidate.preparation?.state === 'preparing' ? 'Preparing automatically' : hostReadinessLabel(candidate.readiness)}</option>{/each}</select></label> <p>Hosts are assigned automatically. We check available capacity before accepting a launch and choose the least loaded eligible host when provisioning starts.</p>
{#if !loading && !hosts.length}<p><a href="/hosts" target="_blank" rel="noopener">Add a host</a>. Its runtime and latest T5 and T6 files will be prepared automatically.</p>{/if} {#if !loading && !hosts.length}<p><a href="/hosts" target="_blank" rel="noopener">Add a host</a>. Its runtime and latest T5 and T6 files will be prepared automatically.</p>{/if}
{#if host}<p>{host.hostname} · ports {host.portMin}–{host.portMax} · {host.publicAddress || host.bindAddress}</p>{/if}
{#if host?.preparation && host.preparation.state !== 'ready'}<p class:error={host.preparation.state === 'blocked'}>{host.preparation.items.find(item => item.state === 'blocked' || item.state === 'preparing' || item.state === 'waiting')?.detail} <a href={`/hosts/${host.id}`} target="_blank" rel="noopener">View automatic setup</a></p>{/if}
{#if !loading}{#each games as game} {#if !loading}{#each games as game}
<div class="game-files"> <div class="game-files">
<strong>{game === 't5' ? 'Black Ops I (T5)' : 'Black Ops II (T6)'} files</strong> <strong>{game === 't5' ? 'Black Ops I (T5)' : 'Black Ops II (T6)'} files</strong>
{#if latest[game]} {#if latest[game]}
<p>Latest upload · {$timeFormat.dateTime(latest[game]?.publishedAt ?? latest[game]?.createdAt)}</p> <p>Latest upload · {$timeFormat.dateTime(latest[game]?.publishedAt ?? latest[game]?.createdAt)}</p>
<p role="status">{!host ? 'Choose a host to check its files.' : cacheReady(game) ? 'Latest files ready on this host.' : 'Waiting for this host to sync the latest files automatically.'}</p> <p role="status">{cacheReady(game) ? 'Latest files ready on an eligible host.' : 'Waiting for a host to prepare the runtime and latest files automatically.'}</p>
{:else} {:else}
<p>No files uploaded yet. <a href="/game-files" target="_blank" rel="noopener">Upload {game.toUpperCase()} files</a> to prepare your hosts.</p> <p>No files uploaded yet. <a href="/game-files" target="_blank" rel="noopener">Upload {game.toUpperCase()} files</a> to prepare your hosts.</p>
{/if} {/if}
@@ -60,11 +57,10 @@
{/each}{/if} {/each}{/if}
<p>Hosts keep the latest T5 and T6 files ready automatically. <a href="/game-files" target="_blank" rel="noopener">Upload game files</a> when they change. Existing servers keep their current files.</p> <p>Hosts keep the latest T5 and T6 files ready automatically. <a href="/game-files" target="_blank" rel="noopener">Upload game files</a> when they change. Existing servers keep their current files.</p>
<details><summary>Advanced: runtime override</summary><p>The configured default runtime is prepared automatically. Custom images must be prepared from the host page.</p><label>Plutainer runtime image<input required bind:value={selection.image} disabled={disabled} onchange={() => onchange?.()} /></label></details> <details><summary>Advanced: runtime override</summary><p>The configured default runtime is prepared automatically. Custom images must be prepared from the host page.</p><label>Plutainer runtime image<input required bind:value={selection.image} disabled={disabled} onchange={() => onchange?.()} /></label></details>
{#if host && !imageReady}<p>Waiting for runtime preparation. <a href={`/hosts/${host.id}`} target="_blank" rel="noopener">View host setup</a>.</p>{/if} {#if network}<details><summary>Network overrides</summary><label>Local bind address<input bind:value={selection.bindAddress} placeholder="Host default" disabled={disabled} onchange={() => onchange?.()} /></label><label>Public / NAT address<input bind:value={selection.publicAddress} placeholder="Host default" disabled={disabled} onchange={() => onchange?.()} /></label><label>Public / NAT port<input type="number" min="1" max="65535" bind:value={selection.publicPort} placeholder="Same as local port" disabled={disabled} onchange={() => onchange?.()} /></label><p>Public mapping does not change the local Docker port reservation.</p></details>{/if}
{#if network}<details><summary>Network overrides</summary><label>Local bind address<input bind:value={selection.bindAddress} placeholder={host?.bindAddress ?? 'Host default'} disabled={disabled} onchange={() => onchange?.()} /></label><label>Public / NAT address<input bind:value={selection.publicAddress} placeholder={host?.publicAddress || 'Host default'} disabled={disabled} onchange={() => onchange?.()} /></label><label>Public / NAT port<input type="number" min="1" max="65535" bind:value={selection.publicPort} placeholder="Same as local port" disabled={disabled} onchange={() => onchange?.()} /></label><p>Public mapping does not change the local Docker port reservation.</p></details>{/if}
</section> </section>
<style> <style>
.game-files { display:grid; gap:5px; } .game-files strong { font-size:13px; font-weight:600; } .game-files { display:grid; gap:5px; } .game-files strong { font-size:13px; font-weight:600; }
.selection { display:grid; gap:14px; padding:20px; border:1px solid var(--border-subtle); border-radius:var(--radius-md); background:var(--surface-sunken); } .title { display:flex; gap:12px; justify-content:space-between; align-items:center; } h3 { font-size:var(--text-sm); margin:0; } label { display:grid; gap:7px; font-size:13px; } input,select { width:100%; box-sizing:border-box; padding:11px; color:var(--text-primary); background:var(--surface); border:1px solid var(--border-control); border-radius:var(--radius-sm); font:inherit; } p { color:var(--text-secondary); font-size:12px; line-height:1.6; margin:0; } a,button { color:var(--accent-primary-hover); } button { background:none; border:0; cursor:pointer; font:inherit; font-size:12px; } button:disabled { opacity:.5; cursor:default; } .error { color:var(--state-danger); } summary { cursor:pointer; font-size:13px; } details label { margin-top:14px; } details p { margin-top:12px; } .selection { display:grid; gap:14px; padding:20px; border:1px solid var(--border-subtle); border-radius:var(--radius-md); background:var(--surface-sunken); } .title { display:flex; gap:12px; justify-content:space-between; align-items:center; } h3 { font-size:var(--text-sm); margin:0; } label { display:grid; gap:7px; font-size:13px; } input { width:100%; box-sizing:border-box; padding:11px; color:var(--text-primary); background:var(--surface); border:1px solid var(--border-control); border-radius:var(--radius-sm); font:inherit; } p { color:var(--text-secondary); font-size:12px; line-height:1.6; margin:0; } a,button { color:var(--accent-primary-hover); } button { background:none; border:0; cursor:pointer; font:inherit; font-size:12px; } button:disabled { opacity:.5; cursor:default; } .error { color:var(--state-danger); } summary { cursor:pointer; font-size:13px; } details label { margin-top:14px; } details p { margin-top:12px; }
</style> </style>
+2 -1
View File
@@ -1,3 +1,4 @@
import { capacityCalibration } from '$lib/server/hosts/calibration.js';
import { error, json } from '@sveltejs/kit'; import { error, json } from '@sveltejs/kit';
import { deleteHost, getHost, hostDeletionStatus, HostDeletionError, listHostOperations, updateHost } from '$lib/server/hosts/index.js'; import { deleteHost, getHost, hostDeletionStatus, HostDeletionError, listHostOperations, updateHost } from '$lib/server/hosts/index.js';
import { hostApiError, objectBody, requireManagementOrigin } from '$lib/server/hosts/http.js'; import { hostApiError, objectBody, requireManagementOrigin } from '$lib/server/hosts/http.js';
@@ -8,7 +9,7 @@ export function GET({ request, params }: Event) {
requireManagementOrigin(request); const host = getHost(params.id); if (!host) error(404, 'Host not found.'); requireManagementOrigin(request); const host = getHost(params.id); if (!host) error(404, 'Host not found.');
// Payloads and results can contain RCON keys or staged credentials. History // Payloads and results can contain RCON keys or staged credentials. History
// exposes metadata only; agent APIs remain the only command delivery path. // exposes metadata only; agent APIs remain the only command delivery path.
return json({ host: presentHostPreparation(host), defaultRuntimeImage: config.komodo.defaultImage, deletion: hostDeletionStatus(host.id), operations: listHostOperations(host.id, 100, true).map(({ payload: _payload, result: _result, ...operation }) => operation) }); return json({ host: presentHostPreparation(host), calibration: capacityCalibration(host.id), defaultRuntimeImage: config.komodo.defaultImage, deletion: hostDeletionStatus(host.id), operations: listHostOperations(host.id, 100, true).map(({ payload: _payload, result: _result, ...operation }) => operation) });
} }
export async function PATCH({ request, params }: Event) { export async function PATCH({ request, params }: Event) {
requireManagementOrigin(request); requireManagementOrigin(request);
+1 -1
View File
@@ -33,7 +33,7 @@ export async function POST({ request }: { request: Request }) {
}); });
if (!result.ok) { if (!result.ok) {
const status = result.kind === 'bad-request' ? 400 : result.kind === 'unconfigured' ? 503 const status = result.kind === 'bad-request' ? 400 : result.kind === 'unconfigured' ? 503
: result.kind === 'unknown-map' || result.kind === 'no-slot' || result.kind === 'no-port' ? 409 : result.kind === 'conflict' || result.kind === 'unknown-map' || result.kind === 'no-slot' || result.kind === 'no-port' ? 409
: 500; : 500;
return json({ ok: false, error: result.message, kind: result.kind }, { status }); return json({ ok: false, error: result.message, kind: result.kind }, { status });
} }
+11 -2
View File
@@ -4,13 +4,14 @@
let requestSequence = 0; let requestSequence = 0;
import { page } from '$app/stores'; import { page } from '$app/stores';
import { goto } from '$app/navigation'; import { goto } from '$app/navigation';
import type { Host, HostOperation, HostMode } from '$lib/hosts.js'; import type { Host, HostOperation, HostMode, CapacityCalibration } from '$lib/hosts.js';
import { hostAgentUpdateLabel, hostAgentUpdateMessage, hostAgentUpdatesEnabled, hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js'; import { hostAgentUpdateLabel, hostAgentUpdateMessage, hostAgentUpdatesEnabled, hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Button from '$ui/components/primitives/Button.svelte'; import Button from '$ui/components/primitives/Button.svelte';
import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte'; import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
import HostPreparation from '$ui/components/HostPreparation.svelte'; import HostPreparation from '$ui/components/HostPreparation.svelte';
let host = $state<Host | null>(null); let operations = $state<HostOperation[]>([]); let host = $state<Host | null>(null); let operations = $state<HostOperation[]>([]);
let calibration = $state<CapacityCalibration | null>(null);
let loading = $state(true); let busy = $state(false); let error = $state(''); let notice = $state(''); let loading = $state(true); let busy = $state(false); let error = $state(''); let notice = $state('');
let deleteOpen = $state(false); let deleteOpen = $state(false);
let name = $state(''); let portMin = $state(28960); let portMax = $state(28980); let bindAddress = $state('0.0.0.0'); let publicAddress = $state(''); let minimumFreeGB = $state(1); let name = $state(''); let portMin = $state(28960); let portMax = $state(28980); let bindAddress = $state('0.0.0.0'); let publicAddress = $state(''); let minimumFreeGB = $state(1);
@@ -22,7 +23,7 @@
async function load(initialize = false, signal?: AbortSignal) { async function load(initialize = false, signal?: AbortSignal) {
const target = endpoint; const sequence = ++requestSequence; const target = endpoint; const sequence = ++requestSequence;
try { try {
const response = await fetch(target, { signal: signal ?? AbortSignal.timeout(15000) }); const body = await response.json(); if (signal?.aborted || target !== endpoint || sequence !== requestSequence) return; if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations; error = ''; const response = await fetch(target, { signal: signal ?? AbortSignal.timeout(15000) }); const body = await response.json(); if (signal?.aborted || target !== endpoint || sequence !== requestSequence) return; if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations; calibration = body.calibration ?? null; error = '';
if (initialize && host) { name = host.name; portMin = host.portMin; portMax = host.portMax; bindAddress = host.bindAddress; publicAddress = host.publicAddress; minimumFreeGB = host.minimumFreeBytes / 1073741824; runtimeImage = body.defaultRuntimeImage ?? runtimeImage; } if (initialize && host) { name = host.name; portMin = host.portMin; portMax = host.portMax; bindAddress = host.bindAddress; publicAddress = host.publicAddress; minimumFreeGB = host.minimumFreeBytes / 1073741824; runtimeImage = body.defaultRuntimeImage ?? runtimeImage; }
} catch (cause) { if (!signal?.aborted && sequence === requestSequence) error = (cause as Error).message; return false; } finally { if (sequence === requestSequence) loading = false; } } catch (cause) { if (!signal?.aborted && sequence === requestSequence) error = (cause as Error).message; return false; } finally { if (sequence === requestSequence) loading = false; }
} }
@@ -55,6 +56,14 @@
<p>Draining prevents new deployments and retires instances as they become safe to remove. Disabled hosts receive no new commands; running game containers continue.</p> <p>Draining prevents new deployments and retires instances as they become safe to remove. Disabled hosts receive no new commands; running game containers continue.</p>
</section> </section>
{#if !cleaning}<section class="panel"><h2>Automatic host setup</h2><HostPreparation {host} /></section>{/if} {#if !cleaning}<section class="panel"><h2>Automatic host setup</h2><HostPreparation {host} /></section>{/if}
{#if calibration}<section class="panel"><h2>Learned server capacity</h2>
<p>Allowances update from observed container usage with 25% headroom. Startup and running usage are tracked separately.</p>
<dl>{#each ['startup', 'running'] as phase}{@const allowance = calibration[phase as keyof CapacityCalibration]}
<dt>{phase === 'startup' ? 'Starting a server' : 'Running a server'}</dt>
<dd>{allowance.cpuCores.toFixed(2)} CPU cores · {bytes(allowance.memoryBytes)} · {allowance.source === 'host' ? 'Learned on this host' : allowance.source === 'fleet' ? 'Learned from the fleet' : 'Calibrating; temporary allowance'} ({allowance.samples} samples)</dd>
{/each}</dl>
{#if calibration.startup.source === 'bootstrap' || calibration.running.source === 'bootstrap'}<p>Learning begins as agent 1.4.3 or later reports running containers. Temporary allowances stay in use until enough measurements are available.</p>{/if}
</section>{/if}
<div class="grid"><section class="panel"><h2>Inventory</h2><dl> <div class="grid"><section class="panel"><h2>Inventory</h2><dl>
<dt>Hostname</dt><dd>{host.hostname || 'Pending'}</dd><dt>Agent</dt><dd>{host.inventory.agentVersion ?? 'Unknown'}</dd><dt>Platform</dt><dd>{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}</dd><dt>CPU</dt><dd>{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)</dd><dt>CPU usage</dt><dd title="Average across all cores since the previous inventory sample">{host.inventory.cpuUsagePercent === undefined ? 'Not reported' : `${host.inventory.cpuUsagePercent.toFixed(1)}%`}{#if !host.connected && host.inventory.cpuUsagePercent !== undefined} (last reported){/if}</dd><dt>Memory available / total</dt><dd>{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}</dd><dt>Game storage available / total</dt><dd>{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}</dd><dt>Docker</dt><dd>{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}</dd><dt>Current operation</dt><dd>{host.inventory.currentOperation ?? 'None reported'}</dd> <dt>Hostname</dt><dd>{host.hostname || 'Pending'}</dd><dt>Agent</dt><dd>{host.inventory.agentVersion ?? 'Unknown'}</dd><dt>Platform</dt><dd>{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}</dd><dt>CPU</dt><dd>{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)</dd><dt>CPU usage</dt><dd title="Average across all cores since the previous inventory sample">{host.inventory.cpuUsagePercent === undefined ? 'Not reported' : `${host.inventory.cpuUsagePercent.toFixed(1)}%`}{#if !host.connected && host.inventory.cpuUsagePercent !== undefined} (last reported){/if}</dd><dt>Memory available / total</dt><dd>{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}</dd><dt>Game storage available / total</dt><dd>{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}</dd><dt>Docker</dt><dd>{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}</dd><dt>Current operation</dt><dd>{host.inventory.currentOperation ?? 'None reported'}</dd>
</dl></section><section class="panel"><h2>Host configuration</h2>{#if canManage}<form onsubmit={save}><fieldset disabled={busy || cleaning}> </dl></section><section class="panel"><h2>Host configuration</h2>{#if canManage}<form onsubmit={save}><fieldset disabled={busy || cleaning}>
+91 -2
View File
@@ -9,6 +9,8 @@ import argparse
import base64 import base64
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
import contextlib import contextlib
from datetime import datetime, timezone
import math
import hashlib import hashlib
import fcntl import fcntl
import json import json
@@ -30,7 +32,7 @@ import urllib.parse
import urllib.request import urllib.request
import uuid import uuid
VERSION = "1.4.1" VERSION = "1.4.3"
LABEL = "io.electric-cherry" LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024 MAX_FILE = 32 * 1024 * 1024
LIBRARY_DOWNLOAD_WORKERS = 4 LIBRARY_DOWNLOAD_WORKERS = 4
@@ -710,6 +712,8 @@ class Agent:
self.docker = docker or Docker() self.docker = docker or Docker()
self.stopping = threading.Event() self.stopping = threading.Event()
self.current_operation = None self.current_operation = None
self.cpu_sample_lock = threading.Lock()
self.cpu_sample = None
self.instance_locks = {} self.instance_locks = {}
self.read_gate = ReadWriteGate() self.read_gate = ReadWriteGate()
self.read_claim_lock = threading.Lock() self.read_claim_lock = threading.Lock()
@@ -899,7 +903,88 @@ class Agent:
except (OSError, ValueError): except (OSError, ValueError):
raise AgentError("Controller request failed") from None raise AgentError("Controller request failed") from None
def cpu_usage_percent(self):
"""Host-wide busy time since the previous inventory sample, without waiting."""
with self.cpu_sample_lock:
previous = self.cpu_sample
self.cpu_sample = None
try:
with Path("/proc/stat").open() as source:
fields = source.readline().split()
if not fields or fields[0] != "cpu" or len(fields) < 9:
return None
# Guest time is already included in user/nice. Idle and I/O
# wait are not busy; steal counts as unavailable CPU capacity.
counters = tuple(int(value) for value in fields[1:9])
if any(value < 0 for value in counters):
return None
except (OSError, ValueError):
return None
self.cpu_sample = counters
if previous is None:
return None
deltas = [current - old for current, old in zip(counters, previous)]
total = sum(deltas)
# Counters can reset and Linux can revise I/O wait downwards.
# Rebase rather than reporting a misleading utilization value.
if total <= 0 or any(delta < 0 for delta in deltas):
return None
return round(100 * (total - deltas[3] - deltas[4]) / total, 1)
def instance_resources(self):
"""One bounded Docker stats batch; unavailable samples are never zero load."""
ids = self.docker.call("container", "ls", "--filter", LABEL + ".managed=true",
"--no-trunc", "--format", "{{.ID}}", timeout=10).splitlines()[:1000]
if not ids:
return []
containers = json.loads(self.docker.call("container", "inspect", *ids, timeout=10))
owned = {}
for info in containers:
try:
labels = info.get("Config", {}).get("Labels", {}) or {}
if labels.get(LABEL + ".managed") != "true" or not info.get("State", {}).get("Running"):
continue
instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", "")))
name = self.container_name(instance_id, generation)
if info.get("Name", "").lstrip("/") != name:
continue
expected = str(self.instance_dir(instance_id, generation) / "app")
if not any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", [])):
continue
started = datetime.fromisoformat(info["State"]["StartedAt"].replace("Z", "+00:00"))
uptime = max(0, int((datetime.now(timezone.utc) - started).total_seconds()))
owned[name] = {"instanceId": instance_id, "generation": generation, "uptimeSeconds": uptime}
except (AgentError, ValueError, KeyError, TypeError):
continue
if not owned:
return []
rows = self.docker.call("stats", "--no-stream", "--no-trunc", "--format", "{{json .}}", *owned, timeout=15)
samples = []
units = {"B": 1, "KiB": 1024, "MiB": 1024 ** 2, "GiB": 1024 ** 3,
"TiB": 1024 ** 4, "kB": 1000, "MB": 1000 ** 2, "GB": 1000 ** 3}
for line in rows.splitlines():
try:
row = json.loads(line)
identity = owned.get(row.get("Name"))
if not identity or int(row["PIDs"]) <= 0:
continue
# Docker Linux CPU percent is core-based: 150% means 1.5 cores.
percent = row["CPUPerc"]
if not percent.endswith("%"):
continue
cores = float(percent[:-1]) / 100
used = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)", row["MemUsage"].split("/", 1)[0].strip())
if not used or used[2] not in units or not math.isfinite(cores) or cores < 0:
continue
memory = math.ceil(float(used[1]) * units[used[2]])
if memory > 0:
samples.append({**identity, "cpuCores": cores, "memoryBytes": memory})
except (ValueError, KeyError, TypeError, AttributeError, OverflowError):
continue
return samples
def inventory(self): def inventory(self):
cpu_usage = self.cpu_usage_percent()
usage = shutil.disk_usage(self.root) usage = shutil.disk_usage(self.root)
memory = {} memory = {}
try: try:
@@ -997,6 +1082,10 @@ class Agent:
if aliases.exists(): if aliases.exists():
with contextlib.suppress(OSError, ValueError): with contextlib.suppress(OSError, ValueError):
images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids) images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids)
resources = {}
if docker_running:
with contextlib.suppress(AgentError, ValueError, TypeError, KeyError):
resources["instanceResources"] = self.instance_resources()
os_name = platform.system() os_name = platform.system()
with contextlib.suppress(OSError): with contextlib.suppress(OSError):
for line in Path("/etc/os-release").read_text().splitlines(): for line in Path("/etc/os-release").read_text().splitlines():
@@ -1004,7 +1093,7 @@ class Agent:
os_name = line.split("=", 1)[1].strip('"') os_name = line.split("=", 1)[1].strip('"')
update = self.update_inventory() update = self.update_inventory()
capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) capabilities = ["instance.cancel-create", "operations.read-lane"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
def safe_to_disrupt(self, instance_id, generation): def safe_to_disrupt(self, instance_id, generation):
health = self.inspect(instance_id, generation) health = self.inspect(instance_id, generation)
+2 -2
View File
@@ -1,8 +1,8 @@
{ {
"version": "1.4.1", "version": "1.4.3",
"protocolVersion": 1, "protocolVersion": 1,
"platform": "debian-amd64", "platform": "debian-amd64",
"sha256": "0691d8457f52ef8b53691b627733352b2f187529a4ecc465cdaaa6e27ddaaaa7", "sha256": "9ca9bff1a42ab838b175329c6a8458e18053429efe2f050f477a9ad9c598410f",
"updaterProtocolVersion": 1, "updaterProtocolVersion": 1,
"updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8" "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
} }
@@ -0,0 +1,129 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
let hostId: string;
const epoch = Date.parse('2026-10-04T12:00:00Z');
const GiB = 1024 ** 3;
const inventory = { cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB,
dockerRunning: true, protocolVersion: 1, directoriesWritable: true, storageAvailable: 100 * GiB,
runtimeImages: ['image'], libraries: [{ id: 'library', state: 'ready' }] };
const sample = { instanceId: 'instance-1', generation: 1, uptimeSeconds: 600, cpuCores: 0.1, memoryBytes: GiB / 4 };
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch);
const hosts = await import('../../src/lib/server/hosts/index.js');
hostId = hosts.createHost({ name: 'First' }).id;
hosts.enrollHost(hosts.createEnrollment(hostId).token, inventory);
await instance(hostId, sample.instanceId);
});
afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); });
async function instance(host: string, id: string) {
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
VALUES(?,?,?,'library','image',28960,?)`).run(id, 't5_zombie_theater_' + id, host, new Date().toISOString());
}
async function report(minute: number, overrides = {}, host = hostId) {
vi.setSystemTime(epoch + minute * 60_000);
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(host, { ...inventory, instanceResources: [{ ...sample, ...overrides }] });
}
async function learned() {
// One real startup followed by twenty running minute peaks.
for (let minute = 0; minute < 4; minute++) await report(minute, { uptimeSeconds: minute * 60, cpuCores: 0.2, memoryBytes: 3 * GiB / 8 });
for (let minute = 5; minute < 25; minute++) await report(minute);
return (await import('../../src/lib/server/hosts/calibration.js')).capacityCalibration(hostId);
}
describe('measured instance capacity', () => {
it('uses temporary allowances until startup and running evidence mature, then replaces both defaults', async () => {
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacityCalibration(hostId).startup).toMatchObject({ source: 'bootstrap', cpuCores: 0.5, memoryBytes: GiB });
const model = await learned();
expect(model.running).toMatchObject({ source: 'host', cpuCores: 0.125, memoryBytes: GiB * 0.3125, samples: 20 });
expect(model.startup).toMatchObject({ source: 'host', cpuCores: 0.25, memoryBytes: GiB * 0.46875, samples: 4 });
});
it('raises allowances immediately on a measured spike, including during cold start', async () => {
await report(0, { cpuCores: 2, memoryBytes: 2 * GiB });
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', cpuCores: 2.5, memoryBytes: 2.5 * GiB });
await learned();
await report(25, { cpuCores: 3, memoryBytes: 3 * GiB });
expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 3.75, memoryBytes: 3.75 * GiB });
expect(capacityCalibration(hostId).startup.cpuCores).toBe(3.75);
});
it('uses recent fleet evidence on new hosts and survives a database restart', async () => {
const model = await learned();
const hosts = await import('../../src/lib/server/hosts/index.js');
const other = hosts.createHost({ name: 'Second' });
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacityCalibration(other.id).running).toMatchObject({ ...model.running, source: 'fleet' });
expect(capacityCalibration(hostId)).toEqual(model);
});
it('prefers mature local measurements when hosts have different CPU footprints', async () => {
await learned();
const hosts = await import('../../src/lib/server/hosts/index.js');
const other = hosts.createHost({ name: 'Slower CPU' });
hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory);
await instance(other.id, 'instance-2');
for (let minute = 25; minute < 45; minute++) await report(minute, { instanceId: 'instance-2', cpuCores: 2 }, other.id);
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacityCalibration(hostId).running).toMatchObject({ source: 'host', cpuCores: 0.125 });
expect(capacityCalibration(other.id).running).toMatchObject({ source: 'host', cpuCores: 2.5 });
expect(capacityCalibration('new-host').running).toMatchObject({ source: 'fleet', cpuCores: 2.5 });
});
it('retains minute peaks instead of treating repeated heartbeats as independent evidence', async () => {
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
for (let n = 0; n < 40; n++) {
vi.setSystemTime(epoch + n * 1000);
heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, cpuCores: n === 0 ? 0.4 : 0.1 }] });
}
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT cpu_cores FROM ec_instance_resource_samples').all()).toEqual([{ cpu_cores: 0.4 }]);
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacityCalibration(hostId).running).toMatchObject({ source: 'bootstrap', samples: 1 });
});
it('does not train from missing, foreign, retired or old-generation samples', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { heartbeat, createHost } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, inventory);
const other = createHost({ name: 'Second' });
await instance(other.id, 'foreign');
await report(0, { instanceId: 'foreign' });
await report(1, { generation: 2 });
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), sample.instanceId);
await report(2);
expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toEqual([]);
});
it('rejects impossible and malformed measurements before changing heartbeat state', async () => {
const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js');
for (const overrides of [{ cpuCores: -1 }, { cpuCores: NaN }, { cpuCores: 17 }, { memoryBytes: 0 },
{ memoryBytes: 65 * GiB }, { generation: 0 }, { uptimeSeconds: -1 }]) {
expect(() => heartbeat(hostId, { ...inventory, instanceResources: [{ ...sample, ...overrides }] })).toThrow('Invalid instance resource');
}
expect(() => heartbeat(hostId, { ...inventory, instanceResources: [sample, sample] })).toThrow('Invalid instance resource');
expect(getHost(hostId)?.inventory.instanceResources).toBeUndefined();
});
it('expires stale models and prunes old measurements as new samples arrive', async () => {
await learned();
vi.setSystemTime(epoch + 26 * 60 * 60_000);
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
await report(26 * 60);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT * FROM ec_instance_resource_samples').all()).toHaveLength(1);
expect(capacityCalibration(hostId).running.source).toBe('bootstrap');
});
it('uses a running percentile and startup peaks with margin, keeping admission reads cached', async () => {
for (let minute = 0; minute < 100; minute++) await report(minute, { cpuCores: minute === 0 ? 2 : 0.1 });
const { db } = await import('../../src/lib/server/db/index.js');
const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
const prepare = vi.spyOn(db(), 'prepare');
expect(capacityCalibration(hostId).running.cpuCores).toBe(0.125);
expect(prepare.mock.calls.some(([sql]) => sql.includes('ec_instance_resource_samples'))).toBe(false);
prepare.mockRestore();
});
});
+180 -1
View File
@@ -13,7 +13,7 @@ vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async ()
let fixture: Fixture; let hostId: string; let libraryId: string; let fixture: Fixture; let hostId: string; let libraryId: string;
const image = 'example.test/plutainer:test'; const image = 'example.test/plutainer:test';
const inventory = () => ({ protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' }] : [] }); const inventory = () => ({ cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] });
beforeEach(async () => { beforeEach(async () => {
vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = ''; vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = '';
fixture = await newFixture([]); fixture = await newFixture([]);
@@ -610,3 +610,182 @@ describe('native instance provisioning', () => {
expect(getServerKey(job.name)).toBe('saved-key'); expect(getServerKey(job.name)).toBe('saved-key');
}); });
}); });
describe('automatic capacity admission and placement', () => {
const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'key' };
async function secondHost(overrides = {}) {
const api = await import('../../src/lib/server/hosts/index.js');
const libs = await import('../../src/lib/server/game-files/index.js');
const host = api.createHost({ name: 'Second host', publicAddress: 'second.example', portMin: 28960, portMax: 28970 });
api.updateHost(host.id, { publicAddress: 'second.example' });
api.enrollHost(api.createEnrollment(host.id).token, { ...inventory(), ...overrides });
const version = libs.getLibraryVersion(libraryId)!;
libs.updateHostLibraryCaches(host.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: version.manifestSha256 }]);
return host;
}
it('admits without a host, then selects the least loaded eligible host and its network defaults', async () => {
await prepareLibrary();
const other = await secondHost({ cpuUsagePercent: 0 });
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
const result = queueCreateServer(input);
expect(result.ok).toBe(true); if (!result.ok) return;
expect(nativeInstance(result.name)?.host_id).toBe(hostId); // Fast first fit.
await advance();
expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id);
expect(nativeInstance(result.name)).toMatchObject({ host_id: other.id, public_address: 'second.example' });
expect(recipeForDeployment(result.deploymentId)?.recipe.native?.hostId).toBe(other.id);
// Once submitted, even a now-offline host cannot trigger another placement.
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(other.id);
await advance();
expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id);
expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
});
it('moves an unsubmitted reservation off an offline host and preserves explicit network overrides', async () => {
await prepareLibrary(); await secondHost();
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
const result = queueNativeCreation({ ...input, publicAddress: 'nat.example', publicPort: 30000 });
expect(result.ok).toBe(true); if (!result.ok) return;
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId);
await advance();
expect(nativeInstance(result.name)).toMatchObject({ public_address: 'nat.example', public_port: 30000 });
expect((await operationFor(result.deploymentId))?.hostId).not.toBe(hostId);
expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toEqual([]);
});
it.each([
{ cpuUsagePercent: undefined }, { memoryAvailable: undefined }, { cpuCount: 0 },
{ cpuUsagePercent: 99 }, { memoryAvailable: 1024 ** 3 }
])('rejects admission with unavailable resources: %j', async overrides => {
await prepareLibrary();
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), ...overrides });
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const { db } = await import('../../src/lib/server/db/index.js');
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict', message: expect.stringContaining('capacity') });
expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
});
it('rechecks preview admission without binding the request to the preview host or port', async () => {
await prepareLibrary();
const other = await secondHost();
const { updateHost } = await import('../../src/lib/server/hosts/index.js');
updateHost(other.id, { portMin: 30000, portMax: 30002 });
const { previewNativeCreation, queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const preview = previewNativeCreation(input);
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId);
const result = queueNativeCreation({ ...input, previewId: preview.id });
expect(result).toMatchObject({ ok: true, name: preview.name, port: 30000 });
expect(queueNativeCreation({ ...input, previewId: preview.id })).toEqual(result);
});
it('rejects an entire collection when capacity shrinks after its preview', async () => {
await prepareLibrary();
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
const preview = previewCollection(blueprint.id);
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
expect(() => queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) })).toThrow('capacity');
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
expect(db().prepare('SELECT * FROM server_launch_collections').all()).toEqual([]);
});
it('uses calibrated footprints to admit more instances than the cold-start allowance', async () => {
const first = await queue();
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
const { recordInstanceResources, capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const measured = { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 };
heartbeat(hostId, measured);
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
const instance = nativeInstance(first.name)!;
for (let minute = 0; minute < 25; minute++) {
recordInstanceResources(hostId, { ...measured, instanceResources: [{ instanceId: instance.id, generation: 1,
uptimeSeconds: minute < 4 ? minute * 60 : 600, cpuCores: 0.1, memoryBytes: 256 * 1024 ** 2 }] },
new Date(Date.now() - (25 - minute) * 60_000).toISOString());
}
expect(capacityCalibration(hostId).startup.source).toBe('host');
expect(queueNativeCreation(input)).toMatchObject({ ok: true });
});
it('charges queued reservations and releases them on cancellation', async () => {
await prepareLibrary();
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
const first = queueNativeCreation(input);
expect(first.ok).toBe(true); if (!first.ok) return;
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
cancelCreation(first.deploymentId);
expect(queueNativeCreation(input)).toMatchObject({ ok: true });
});
it('waits when capacity disappears after admission without sending a create command', async () => {
await prepareLibrary();
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const first = queueNativeCreation(input);
expect(first.ok).toBe(true); if (!first.ok) return;
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 });
await advance();
expect(await operationFor(first.deploymentId)).toBeNull();
expect((await detail(first.deploymentId)).stage).toContain('capacity');
});
it('uses recent history at placement while keeping admission to current inventory', async () => {
await prepareLibrary();
const other = await secondHost({ cpuUsagePercent: 30 });
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available)
VALUES(?,?,16,95,?,?)`).run(hostId, new Date(Date.now() - 60_000).toISOString(), 64 * 1024 ** 3, 60 * 1024 ** 3);
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const first = queueNativeCreation(input);
expect(first.ok).toBe(true); if (!first.ok) return;
await advance();
expect((await operationFor(first.deploymentId))?.hostId).toBe(other.id);
});
it('persists nullable CPU/memory samples and prunes old history per host', async () => {
await prepareLibrary();
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
db().prepare("INSERT INTO ec_host_resource_samples(host_id,sampled_at) VALUES(?,'2000-01-01')").run(hostId);
heartbeat(hostId, { ...inventory(), cpuUsagePercent: undefined });
closeDb();
const rows = db().prepare('SELECT * FROM ec_host_resource_samples WHERE host_id=? ORDER BY sampled_at').all(hostId) as { cpu_usage_percent: number | null; memory_available: number; sampled_at: string }[];
expect(rows.at(-1)).toMatchObject({ cpu_usage_percent: null, memory_available: 60 * 1024 ** 3 });
expect(rows.some(row => row.sampled_at === '2000-01-01')).toBe(false);
});
it('checks both port protocols and skips draining hosts', async () => {
await prepareLibrary();
const other = await secondHost();
const { reserveHostPort, updateHost } = await import('../../src/lib/server/hosts/index.js');
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
for (let port = 28960; port <= 28970; port++) reserveHostPort(hostId, 'external-' + port, { port, protocol: 'tcp' });
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
const result = queueNativeCreation(input);
expect(result.ok).toBe(true); if (!result.ok) return;
expect(nativeInstance(result.name)?.host_id).toBe(other.id);
updateHost(other.id, { mode: 'draining' });
expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
});
it('spreads collection admission across hosts, allowing the same port on different hosts', async () => {
await prepareLibrary(); await secondHost({ cpuCount: 1, cpuUsagePercent: 0 });
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
const preview = previewCollection(blueprint.id);
expect(preview.ready).toBe(true);
expect(preview.servers.map(server => server.port)).toEqual([28960, 28960]);
expect(new Set(preview.servers.map(server => server.native?.hostId)).size).toBe(2);
expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) }).ok).toBe(true);
const { db } = await import('../../src/lib/server/db/index.js');
expect(db().prepare('SELECT DISTINCT host_id FROM native_instances').all()).toHaveLength(2);
});
});