Fix container resource sampling and surface collection failures

agent/ec_agent.py

@@ -32,7 +32,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.5"
+VERSION = "1.4.6"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 LIBRARY_DOWNLOAD_WORKERS = 4
@@ -933,7 +933,7 @@ class Agent:
 
     def instance_resources(self):
         """One bounded Docker stats batch; unavailable samples are never zero load."""
-        ids = self.docker.call("container", "ls", "--filter", LABEL + ".managed=true",
+        ids = self.docker.call("container", "ls", "--filter", "label=" + LABEL + ".managed=true",
                                "--no-trunc", "--format", "{{.ID}}", timeout=10).splitlines()[:1000]
         if not ids:
             return []
@@ -1084,8 +1084,10 @@ class Agent:
                         images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids)
         resources = {}
         if docker_running:
-            with contextlib.suppress(AgentError, ValueError, TypeError, KeyError):
+            try:
                 resources["instanceResources"] = self.instance_resources()
+            except (AgentError, ValueError, TypeError, KeyError):
+                resources["instanceResourcesError"] = "Container resource sampling failed. Check Docker access and stats availability on this host."
         os_name = platform.system()
         with contextlib.suppress(OSError):
             for line in Path("/etc/os-release").read_text().splitlines():

agent/tests/test_agent.py

@@ -124,6 +124,7 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(samples[0]["cpuCores"], 1.5)
         self.assertEqual(samples[0]["memoryBytes"], int(384.5 * 1024 ** 2))
         self.assertGreater(samples[0]["uptimeSeconds"], 300)
+        self.assertEqual(call.call_args_list[0].args, ("container", "ls", "--filter", "label=io.electric-cherry.managed=true", "--no-trunc", "--format", "{{.ID}}"))
         self.assertEqual(call.call_args_list[-1].args, ("stats", "--no-stream", "--no-trunc", "--format", "{{json .}}", name))
 
     def test_instance_resources_ignores_stopped_foreign_and_mismatched_containers(self):
@@ -149,7 +150,13 @@ class AgentTests(unittest.TestCase):
         with patch.object(self.docker, "call", side_effect=["container-id\n", json.dumps([self.resource_container()]), json.dumps(good)]):
             self.assertEqual(self.agent.instance_resources()[0]["cpuCores"], 0)
         with patch.object(self.agent, "instance_resources", side_effect=AgentError("timed out")):
-            self.assertNotIn("instanceResources", self.agent.inventory())
+            inventory = self.agent.inventory()
+            self.assertNotIn("instanceResources", inventory)
+            self.assertIn("sampling failed", inventory["instanceResourcesError"])
+        with patch.object(self.agent, "instance_resources", return_value=[]):
+            inventory = self.agent.inventory()
+            self.assertEqual(inventory["instanceResources"], [])
+            self.assertNotIn("instanceResourcesError", inventory)
 
     def operation(self, kind, payload=None, op_id="operation-1", generation=1):
         return {"id": op_id, "kind": kind, "instanceId": INSTANCE, "generation": generation, "payload": payload or {}}

docs/native-hosts.md

@@ -140,12 +140,19 @@ memory measurements. Hosts with stale or missing measurements cannot accept new
 servers. Existing API clients may still explicitly constrain a launch to a host.
 
 Admission reserves a provisional host and port atomically, including collection
-launches. Agent **1.4.3** samples each managed running container with a bounded
+launches. Agent **1.4.6** fixes container discovery for resource sampling (the
+earlier collector omitted `label=` from its Docker filter). It samples each
+managed running container with a bounded
 [`docker stats`](https://docs.docker.com/reference/cli/docker/container/stats/)
 batch during inventory collection. CPU is normalized to cores (100% Docker CPU
 is one core); memory is the Linux working usage reported by Docker, excluding
 reclaimable cache. Missing or failed samples stay unknown instead of becoming
 zero usage. Samples must match the host's live instance and generation.
+Measurements arrive through `/api/agent/heartbeat`, normally every 15 seconds
+plus collection and delivery time, independently of RCON or inspection operations.
+Sampling failures appear on the host's Learned server capacity panel. Existing
+containers can begin contributing running samples as soon as the agent updates;
+startup samples require observing the first five minutes of a container's run.
 
 The controller retains per-instance minute peaks for 24 hours and caches learned
 CPU/memory allowances. The first five minutes after a container starts count as

src/lib/hosts.ts

@@ -69,6 +69,7 @@ export interface HostInventory {
     capabilities?: string[];
     agentUpdate?: HostAgentUpdate;
     instanceResources?: InstanceResources[];
+    instanceResourcesError?: string;
 }
 export interface InstanceResources {
     instanceId: string; generation: number; uptimeSeconds: number; cpuCores: number; memoryBytes: number;

src/lib/server/hosts/calibration.ts

@@ -15,6 +15,8 @@ interface AllowanceRow {
 function database() { const handle = db(); ensureHostSchema(handle); return handle; }
 
 export function validateInstanceResources(inventory: HostInventory): void {
+    if (inventory.instanceResourcesError !== undefined && (typeof inventory.instanceResourcesError !== 'string'
+        || inventory.instanceResourcesError.length > 500)) throw new Error('Invalid instance resource sampling error.');
     const samples = inventory.instanceResources;
     if (samples === undefined) return;
     if (!Array.isArray(samples) || samples.length > 1000) throw new Error('Invalid instance resource samples.');

src/routes/hosts/[id]/+page.svelte

@@ -62,7 +62,8 @@
                 <dt>{phase === 'startup' ? 'Starting a server' : 'Running a server'}</dt>
                 <dd>{allowance.cpuCores.toFixed(2)} CPU cores · {bytes(allowance.memoryBytes)} · {allowance.source === 'host' ? 'Learned on this host' : allowance.source === 'fleet' ? 'Learned from the fleet' : 'Calibrating; temporary allowance'} ({allowance.samples} samples)</dd>
             {/each}</dl>
-            {#if calibration.startup.source === 'bootstrap' || calibration.running.source === 'bootstrap'}<p>Learning begins as agent 1.4.3 or later reports running containers. Temporary allowances stay in use until enough measurements are available.</p>{/if}
+            {#if host.inventory.instanceResourcesError}<p role="status">{host.inventory.instanceResourcesError}</p>{/if}
+            {#if calibration.startup.source === 'bootstrap' || calibration.running.source === 'bootstrap'}<p>Use agent 1.4.6 or later for resource sampling. Measurements arrive with agent heartbeats and are combined into one peak sample per container per minute. The first five minutes after a container starts count toward startup; later measurements count toward running. Temporary allowances stay in use until enough measurements are available.</p>{/if}
         </section>{/if}
         <div class="grid"><section class="panel"><h2>Inventory</h2><dl>
             <dt>Hostname</dt><dd>{host.hostname || 'Pending'}</dd><dt>Agent</dt><dd>{host.inventory.agentVersion ?? 'Unknown'}</dd><dt>Platform</dt><dd>{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}</dd><dt>CPU</dt><dd>{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)</dd><dt>CPU usage</dt><dd title="Average across all cores since the previous inventory sample">{host.inventory.cpuUsagePercent === undefined ? 'Not reported' : `${host.inventory.cpuUsagePercent.toFixed(1)}%`}{#if !host.connected && host.inventory.cpuUsagePercent !== undefined} (last reported){/if}</dd><dt>Memory available / total</dt><dd>{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}</dd><dt>Game storage available / total</dt><dd>{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}</dd><dt>Docker</dt><dd>{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}</dd><dt>Current operation</dt><dd>{host.inventory.currentOperation ?? 'None reported'}</dd>

static/agent/ec-agent.py

@@ -32,7 +32,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.5"
+VERSION = "1.4.6"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 LIBRARY_DOWNLOAD_WORKERS = 4
@@ -933,7 +933,7 @@ class Agent:
 
     def instance_resources(self):
         """One bounded Docker stats batch; unavailable samples are never zero load."""
-        ids = self.docker.call("container", "ls", "--filter", LABEL + ".managed=true",
+        ids = self.docker.call("container", "ls", "--filter", "label=" + LABEL + ".managed=true",
                                "--no-trunc", "--format", "{{.ID}}", timeout=10).splitlines()[:1000]
         if not ids:
             return []
@@ -1084,8 +1084,10 @@ class Agent:
                         images.extend(alias for alias, image_id in json.loads(aliases.read_text()).items() if image_id in image_ids)
         resources = {}
         if docker_running:
-            with contextlib.suppress(AgentError, ValueError, TypeError, KeyError):
+            try:
                 resources["instanceResources"] = self.instance_resources()
+            except (AgentError, ValueError, TypeError, KeyError):
+                resources["instanceResourcesError"] = "Container resource sampling failed. Check Docker access and stats availability on this host."
         os_name = platform.system()
         with contextlib.suppress(OSError):
             for line in Path("/etc/os-release").read_text().splitlines():

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.4.5",
+  "version": "1.4.6",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "bd4afa2ec3ab6a3022dba99d0580b4b6b1ba5622ae4138035e3e3862885a97bd",
+  "sha256": "bdedda91475bd29c2140bf32932a1dfc358e7069736439b036ffb333b4300cc3",
   "updaterProtocolVersion": 1,
   "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }

tests/unit/host-capacity-calibration.test.ts

@@ -38,6 +38,21 @@ async function learned() {
 }
 
 describe('measured instance capacity', () => {
+    it('reports sampling failure without inventing measurements and clears it after recovery', async () => {
+        const { heartbeat, getHost } = await import('../../src/lib/server/hosts/index.js');
+        const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
+        const error = 'Container resource sampling failed.';
+        heartbeat(hostId, { ...inventory, instanceResourcesError: error });
+        expect(getHost(hostId)?.inventory.instanceResourcesError).toBe(error);
+        expect(capacityCalibration(hostId).running.samples).toBe(0);
+        for (const invalid of [42, {}, 'x'.repeat(501)]) {
+            expect(() => heartbeat(hostId, { ...inventory, instanceResourcesError: invalid })).toThrow('Invalid instance resource sampling error');
+        }
+        await report(0);
+        expect(getHost(hostId)?.inventory.instanceResourcesError).toBeUndefined();
+        expect(capacityCalibration(hostId).running.samples).toBe(1);
+    });
+
     it('uses temporary allowances until startup and running evidence mature, then replaces both defaults', async () => {
         const { capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
         expect(capacityCalibration(hostId).startup).toMatchObject({ source: 'bootstrap', cpuCores: 0.5, memoryBytes: GiB });