Keep invalid container states from dropping host heartbeats

agent/ec_agent.py

@@ -32,7 +32,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.11"
+VERSION = "1.4.12"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1054,7 +1054,7 @@ class Agent:
             raise AgentError("Container ownership or generation does not match")
         state = info.get("State", {})
         status = state.get("Status", "unknown")
-        healthy = state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy"
+        healthy = state.get("Status") == "running" and state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy"
         if status == "running" and not healthy:
             status = state.get("Health", {}).get("Status", "unknown")
         return {"exists": True, "running": state.get("Running") is True, "healthy": healthy, "state": status, "containerId": info.get("Id"), "instanceId": instance_id, "generation": generation, "startedAt": state.get("StartedAt"), "creationOperation": labels.get(LABEL + ".operation"), "specSha256": labels.get(LABEL + ".spec-sha256")}
@@ -1152,7 +1152,7 @@ class Agent:
                 if not any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", [])):
                     continue
                 state = info.get("State", {})
-                healthy = state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy"
+                healthy = state.get("Status") == "running" and state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy"
                 status = state.get("Status", "unknown")
                 if status == "running" and not healthy:
                     status = state.get("Health", {}).get("Status", "unknown")

agent/tests/test_agent.py

@@ -583,7 +583,7 @@ class AgentTests(unittest.TestCase):
             self.assertNotIn("instanceResourcesError", inventory)
 
     def test_heartbeat_states_share_inspection_and_include_stopped_and_unhealthy_containers(self):
-        for status, running, health in (("exited", False, "healthy"), ("running", True, "unhealthy"), ("running", True, "healthy")):
+        for status, running, health in (("exited", False, "healthy"), ("running", True, "unhealthy"), ("running", True, "healthy"), ("restarting", True, "healthy"), ("paused", True, "healthy")):
             info = self.resource_container()
             info["State"].update({"Status": status, "Running": running, "Health": {"Status": health}})
             observations = []
@@ -593,7 +593,9 @@ class AgentTests(unittest.TestCase):
             self.assertEqual(observations[0]["instanceId"], INSTANCE)
             self.assertEqual(observations[0]["generation"], 1)
             self.assertEqual(observations[0]["state"], health if running and health != "healthy" else status)
-            self.assertEqual(observations[0]["healthy"], running and health == "healthy")
+            self.assertEqual(observations[0]["healthy"], status == "running" and running and health == "healthy")
+            self.docker.containers[self.agent.container_name(INSTANCE, 1)] = info
+            self.assertEqual(self.agent.inspect(INSTANCE, 1)["healthy"], observations[0]["healthy"])
             self.assertIn("observedAt", observations[0])
             self.assertEqual(sum(args.args[:2] == ("container", "inspect") for args in docker.call_args_list), 1)
         info["Mounts"][0]["Source"] = "/foreign/app"

src/lib/server/hosts/index.ts

@@ -255,14 +255,17 @@ export function validateInventory(value: unknown): HostInventory {
     }
     if (inventory.cpuUsagePercent !== undefined && (typeof inventory.cpuUsagePercent !== 'number' || !Number.isFinite(inventory.cpuUsagePercent) || inventory.cpuUsagePercent < 0 || inventory.cpuUsagePercent > 100)) throw new Error('Invalid inventory cpuUsagePercent.');
     validateInstanceResources(inventory);
-    if (inventory.instanceStates !== undefined && (!Array.isArray(inventory.instanceStates) || inventory.instanceStates.length > 1000
+    if (inventory.instanceStates !== undefined) {
-        || inventory.instanceStates.some(state => !state || typeof state !== 'object'
+        if (!Array.isArray(inventory.instanceStates) || inventory.instanceStates.length > 1000) {
-            || typeof state.instanceId !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(state.instanceId)
+            throw new Error('Invalid instance state inventory.');
-            || !Number.isSafeInteger(state.generation) || state.generation < 1
+        }
-            || typeof state.state !== 'string' || !/^[a-z-]{1,40}$/.test(state.state)
+        // A malformed server observation must not discard the host heartbeat.
-            || typeof state.healthy !== 'boolean' || state.healthy && state.state !== 'running'
+        inventory.instanceStates = inventory.instanceStates.filter(state => state && typeof state === 'object'
-            || typeof state.observedAt !== 'string' || state.observedAt.length > 40 || !Number.isFinite(Date.parse(state.observedAt))))) {
+            && typeof state.instanceId === 'string' && /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(state.instanceId)
-        throw new Error('Invalid instance state inventory.');
+            && Number.isSafeInteger(state.generation) && state.generation >= 1
+            && typeof state.state === 'string' && /^[a-z-]{1,40}$/.test(state.state)
+            && typeof state.healthy === 'boolean' && (!state.healthy || state.state === 'running')
+            && typeof state.observedAt === 'string' && state.observedAt.length <= 40 && Number.isFinite(Date.parse(state.observedAt)));
     }
     for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
     if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');

static/agent/ec-agent.py

@@ -32,7 +32,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.4.11"
+VERSION = "1.4.12"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1054,7 +1054,7 @@ class Agent:
             raise AgentError("Container ownership or generation does not match")
         state = info.get("State", {})
         status = state.get("Status", "unknown")
-        healthy = state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy"
+        healthy = state.get("Status") == "running" and state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy"
         if status == "running" and not healthy:
             status = state.get("Health", {}).get("Status", "unknown")
         return {"exists": True, "running": state.get("Running") is True, "healthy": healthy, "state": status, "containerId": info.get("Id"), "instanceId": instance_id, "generation": generation, "startedAt": state.get("StartedAt"), "creationOperation": labels.get(LABEL + ".operation"), "specSha256": labels.get(LABEL + ".spec-sha256")}
@@ -1152,7 +1152,7 @@ class Agent:
                 if not any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", [])):
                     continue
                 state = info.get("State", {})
-                healthy = state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy"
+                healthy = state.get("Status") == "running" and state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy"
                 status = state.get("Status", "unknown")
                 if status == "running" and not healthy:
                     status = state.get("Health", {}).get("Status", "unknown")

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.4.11",
+  "version": "1.4.12",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "4c6cc180f3c45a436878a0278d62b6840526150e039fdf51bafadef63759b131",
+  "sha256": "1e1bf2f04c02f0c8d0db8e4dbf39fadddc39c8bd04db5b8db753b6fe9e04c315",
   "updaterProtocolVersion": 1,
   "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }

tests/unit/native-hosts.test.ts

@@ -15,6 +15,19 @@ async function enrolled() {
 }
 
 describe('native host enrollment and scoped authentication', () => {
+    it('keeps the host online and delivers work when individual state observations are invalid', async () => {
+        const api = await enrolled();
+        const good = { instanceId: 'healthy-server', generation: 1, state: 'running', healthy: true, observedAt: new Date().toISOString() };
+        const operation = api.queueHostOperation(api.host.id, 'host.inspect', {});
+        const host = api.heartbeat(api.host.id, { ...inventory, instanceStates: [good,
+            { ...good, instanceId: 'crashing', state: 'restarting' },
+            { ...good, instanceId: 'paused', state: 'paused' }, null,
+            { ...good, generation: 0 }, { ...good, observedAt: 'invalid' }] });
+        expect(host.inventory.instanceStates).toEqual([good]);
+        expect(host.readiness).toBe('ready');
+        expect(api.pendingHostOperations(api.host.id).map(row => row.id)).toContain(operation.id);
+    });
+
     it('queues serialized launch files and processes receipts without hydrating plugin payloads', async () => {
         const api = await enrolled();
         const { db } = await import('../../src/lib/server/db/index.js');