From 148c74063bfaeb375d41b949b3d01dda32d3f977 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Sun, 4 Oct 2026 22:02:03 -0400 Subject: [PATCH] Fix container resource sampling and surface collection failures --- agent/ec_agent.py | 8 +++++--- agent/tests/test_agent.py | 9 ++++++++- docs/native-hosts.md | 9 ++++++++- src/lib/hosts.ts | 1 + src/lib/server/hosts/calibration.ts | 2 ++ src/routes/hosts/[id]/+page.svelte | 3 ++- static/agent/ec-agent.py | 8 +++++--- static/agent/manifest.json | 4 ++-- tests/unit/host-capacity-calibration.test.ts | 15 +++++++++++++++ 9 files changed, 48 insertions(+), 11 deletions(-) diff --git a/agent/ec_agent.py b/agent/ec_agent.py index 3079a19..474dd34 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -32,7 +32,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.4.5" +VERSION = "1.4.6" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 LIBRARY_DOWNLOAD_WORKERS = 4 @@ -933,7 +933,7 @@ class Agent: def instance_resources(self): """One bounded Docker stats batch; unavailable samples are never zero load.""" - ids = self.docker.call("container", "ls", "--filter", LABEL + ".managed=true", + ids = self.docker.call("container", "ls", "--filter", "label=" + LABEL + ".managed=true", "--no-trunc", "--format", "{{.ID}}", timeout=10).splitlines()[:1000] if not ids: return [] @@ -1084,8 +1084,10 @@ class Agent: images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids) resources = {} if docker_running: - with contextlib.suppress(AgentError, ValueError, TypeError, KeyError): + try: resources["instanceResources"] = self.instance_resources() + except (AgentError, ValueError, TypeError, KeyError): + resources["instanceResourcesError"] = "Container resource sampling failed. Check Docker access and stats availability on this host." os_name = platform.system() with contextlib.suppress(OSError): for line in Path("/etc/os-release").read_text().splitlines(): diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index dab0e0a..f15fc40 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -124,6 +124,7 @@ class AgentTests(unittest.TestCase): self.assertEqual(samples[0]["cpuCores"], 1.5) self.assertEqual(samples[0]["memoryBytes"], int(384.5 * 1024 ** 2)) self.assertGreater(samples[0]["uptimeSeconds"], 300) + self.assertEqual(call.call_args_list[0].args, ("container", "ls", "--filter", "label=io.electric-cherry.managed=true", "--no-trunc", "--format", "{{.ID}}")) self.assertEqual(call.call_args_list[-1].args, ("stats", "--no-stream", "--no-trunc", "--format", "{{json .}}", name)) def test_instance_resources_ignores_stopped_foreign_and_mismatched_containers(self): @@ -149,7 +150,13 @@ class AgentTests(unittest.TestCase): with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([self.resource_container()]), json.dumps(good)]): self.assertEqual(self.agent.instance_resources()[0]["cpuCores"], 0) with patch.object(self.agent, "instance_resources", side_effect=AgentError("timed out")): - self.assertNotIn("instanceResources", self.agent.inventory()) + inventory = self.agent.inventory() + self.assertNotIn("instanceResources", inventory) + self.assertIn("sampling failed", inventory["instanceResourcesError"]) + with patch.object(self.agent, "instance_resources", return_value=[]): + inventory = self.agent.inventory() + self.assertEqual(inventory["instanceResources"], []) + self.assertNotIn("instanceResourcesError", inventory) def operation(self, kind, payload=None, op_id="operation-1", generation=1): return {"id": op_id, "kind": kind, "instanceId": INSTANCE, "generation": generation, "payload": payload or {}} diff --git a/docs/native-hosts.md b/docs/native-hosts.md index 0067f3a..676906b 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -140,12 +140,19 @@ memory measurements. Hosts with stale or missing measurements cannot accept new servers. Existing API clients may still explicitly constrain a launch to a host. Admission reserves a provisional host and port atomically, including collection -launches. Agent **1.4.3** samples each managed running container with a bounded +launches. Agent **1.4.6** fixes container discovery for resource sampling (the +earlier collector omitted `label=` from its Docker filter). It samples each +managed running container with a bounded [`docker stats`](https://docs.docker.com/reference/cli/docker/container/stats/) batch during inventory collection. CPU is normalized to cores (100% Docker CPU is one core); memory is the Linux working usage reported by Docker, excluding reclaimable cache. Missing or failed samples stay unknown instead of becoming zero usage. Samples must match the host's live instance and generation. +Measurements arrive through `/api/agent/heartbeat`, normally every 15 seconds +plus collection and delivery time, independently of RCON or inspection operations. +Sampling failures appear on the host's Learned server capacity panel. Existing +containers can begin contributing running samples as soon as the agent updates; +startup samples require observing the first five minutes of a container's run. The controller retains per-instance minute peaks for 24 hours and caches learned CPU/memory allowances. The first five minutes after a container starts count as diff --git a/src/lib/hosts.ts b/src/lib/hosts.ts index 2844e28..303fbcf 100644 --- a/src/lib/hosts.ts +++ b/src/lib/hosts.ts @@ -69,6 +69,7 @@ export interface HostInventory { capabilities?: string[]; agentUpdate?: HostAgentUpdate; instanceResources?: InstanceResources[]; + instanceResourcesError?: string; } export interface InstanceResources { instanceId: string; generation: number; uptimeSeconds: number; cpuCores: number; memoryBytes: number; diff --git a/src/lib/server/hosts/calibration.ts b/src/lib/server/hosts/calibration.ts index c2726c0..ef948c8 100644 --- a/src/lib/server/hosts/calibration.ts +++ b/src/lib/server/hosts/calibration.ts @@ -15,6 +15,8 @@ interface AllowanceRow { function database() { const handle = db(); ensureHostSchema(handle); return handle; } export function validateInstanceResources(inventory: HostInventory): void { + if (inventory.instanceResourcesError !== undefined && (typeof inventory.instanceResourcesError !== 'string' + || inventory.instanceResourcesError.length > 500)) throw new Error('Invalid instance resource sampling error.'); const samples = inventory.instanceResources; if (samples === undefined) return; if (!Array.isArray(samples) || samples.length > 1000) throw new Error('Invalid instance resource samples.'); diff --git a/src/routes/hosts/[id]/+page.svelte b/src/routes/hosts/[id]/+page.svelte index ade8eb5..c3e0a71 100644 --- a/src/routes/hosts/[id]/+page.svelte +++ b/src/routes/hosts/[id]/+page.svelte @@ -62,7 +62,8 @@
{phase === 'startup' ? 'Starting a server' : 'Running a server'}
{allowance.cpuCores.toFixed(2)} CPU cores · {bytes(allowance.memoryBytes)} · {allowance.source === 'host' ? 'Learned on this host' : allowance.source === 'fleet' ? 'Learned from the fleet' : 'Calibrating; temporary allowance'} ({allowance.samples} samples)
{/each} - {#if calibration.startup.source === 'bootstrap' || calibration.running.source === 'bootstrap'}

Learning begins as agent 1.4.3 or later reports running containers. Temporary allowances stay in use until enough measurements are available.

{/if} + {#if host.inventory.instanceResourcesError}

{host.inventory.instanceResourcesError}

{/if} + {#if calibration.startup.source === 'bootstrap' || calibration.running.source === 'bootstrap'}

Use agent 1.4.6 or later for resource sampling. Measurements arrive with agent heartbeats and are combined into one peak sample per container per minute. The first five minutes after a container starts count toward startup; later measurements count toward running. Temporary allowances stay in use until enough measurements are available.

{/if} {/if}

Inventory

Hostname
{host.hostname || 'Pending'}
Agent
{host.inventory.agentVersion ?? 'Unknown'}
Platform
{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}
CPU
{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)
CPU usage
{host.inventory.cpuUsagePercent === undefined ? 'Not reported' : `${host.inventory.cpuUsagePercent.toFixed(1)}%`}{#if !host.connected && host.inventory.cpuUsagePercent !== undefined} (last reported){/if}
Memory available / total
{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}
Game storage available / total
{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}
Docker
{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}
Current operation
{host.inventory.currentOperation ?? 'None reported'}
diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py index 3079a19..474dd34 100644 --- a/static/agent/ec-agent.py +++ b/static/agent/ec-agent.py @@ -32,7 +32,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.4.5" +VERSION = "1.4.6" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 LIBRARY_DOWNLOAD_WORKERS = 4 @@ -933,7 +933,7 @@ class Agent: def instance_resources(self): """One bounded Docker stats batch; unavailable samples are never zero load.""" - ids = self.docker.call("container", "ls", "--filter", LABEL + ".managed=true", + ids = self.docker.call("container", "ls", "--filter", "label=" + LABEL + ".managed=true", "--no-trunc", "--format", "{{.ID}}", timeout=10).splitlines()[:1000] if not ids: return [] @@ -1084,8 +1084,10 @@ class Agent: images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids) resources = {} if docker_running: - with contextlib.suppress(AgentError, ValueError, TypeError, KeyError): + try: resources["instanceResources"] = self.instance_resources() + except (AgentError, ValueError, TypeError, KeyError): + resources["instanceResourcesError"] = "Container resource sampling failed. Check Docker access and stats availability on this host." os_name = platform.system() with contextlib.suppress(OSError): for line in Path("/etc/os-release").read_text().splitlines(): diff --git a/static/agent/manifest.json b/static/agent/manifest.json index 06bff47..48df62d 100644 --- a/static/agent/manifest.json +++ b/static/agent/manifest.json @@ -1,8 +1,8 @@ { - "version": "1.4.5", + "version": "1.4.6", "protocolVersion": 1, "platform": "debian-amd64", - "sha256": "bd4afa2ec3ab6a3022dba99d0580b4b6b1ba5622ae4138035e3e3862885a97bd", + "sha256": "bdedda91475bd29c2140bf32932a1dfc358e7069736439b036ffb333b4300cc3", "updaterProtocolVersion": 1, "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8" } diff --git a/tests/unit/host-capacity-calibration.test.ts b/tests/unit/host-capacity-calibration.test.ts index 4c20770..b629ddc 100644 --- a/tests/unit/host-capacity-calibration.test.ts +++ b/tests/unit/host-capacity-calibration.test.ts @@ -38,6 +38,21 @@ async function learned() { } describe('measured instance capacity', () => { + it('reports sampling failure without inventing measurements and clears it after recovery', async () => { + const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js'); + const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); + const error = 'Container resource sampling failed.'; + heartbeat(hostId, { ...inventory, instanceResourcesError: error }); + expect(getHost(hostId)?.inventory.instanceResourcesError).toBe(error); + expect(capacityCalibration(hostId).running.samples).toBe(0); + for (const invalid of [42, {}, 'x'.repeat(501)]) { + expect(() => heartbeat(hostId, { ...inventory, instanceResourcesError: invalid })).toThrow('Invalid instance resource sampling error'); + } + await report(0); + expect(getHost(hostId)?.inventory.instanceResourcesError).toBeUndefined(); + expect(capacityCalibration(hostId).running.samples).toBe(1); + }); + it('uses temporary allowances until startup and running evidence mature, then replaces both defaults', async () => { const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js'); expect(capacityCalibration(hostId).startup).toMatchObject({ source: 'bootstrap', cpuCores: 0.5, memoryBytes: GiB });