From ad63b3957d676c8af451bb3c889f590d58c58f43 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 5 Oct 2026 16:15:15 -0400 Subject: [PATCH] Keep invalid container states from dropping host heartbeats --- agent/ec_agent.py | 6 +++--- agent/tests/test_agent.py | 6 ++++-- src/lib/server/hosts/index.ts | 19 +++++++++++-------- static/agent/ec-agent.py | 6 +++--- static/agent/manifest.json | 4 ++-- tests/unit/native-hosts.test.ts | 13 +++++++++++++ 6 files changed, 36 insertions(+), 18 deletions(-) diff --git a/agent/ec_agent.py b/agent/ec_agent.py index f195d92..6297f00 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -32,7 +32,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.4.11" +VERSION = "1.4.12" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -1054,7 +1054,7 @@ class Agent: raise AgentError("Container ownership or generation does not match") state = info.get("State", {}) status = state.get("Status", "unknown") - healthy = state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy" + healthy = state.get("Status") == "running" and state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy" if status == "running" and not healthy: status = state.get("Health", {}).get("Status", "unknown") return {"exists": True, "running": state.get("Running") is True, "healthy": healthy, "state": status, "containerId": info.get("Id"), "instanceId": instance_id, "generation": generation, "startedAt": state.get("StartedAt"), "creationOperation": labels.get(LABEL + ".operation"), "specSha256": labels.get(LABEL + ".spec-sha256")} @@ -1152,7 +1152,7 @@ class Agent: if not any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", [])): continue state = info.get("State", {}) - healthy = state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy" + healthy = state.get("Status") == "running" and state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy" status = state.get("Status", "unknown") if status == "running" and not healthy: status = state.get("Health", {}).get("Status", "unknown") diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 3f5c1db..90cbb58 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -583,7 +583,7 @@ class AgentTests(unittest.TestCase): self.assertNotIn("instanceResourcesError", inventory) def test_heartbeat_states_share_inspection_and_include_stopped_and_unhealthy_containers(self): - for status, running, health in (("exited", False, "healthy"), ("running", True, "unhealthy"), ("running", True, "healthy")): + for status, running, health in (("exited", False, "healthy"), ("running", True, "unhealthy"), ("running", True, "healthy"), ("restarting", True, "healthy"), ("paused", True, "healthy")): info = self.resource_container() info["State"].update({"Status": status, "Running": running, "Health": {"Status": health}}) observations = [] @@ -593,7 +593,9 @@ class AgentTests(unittest.TestCase): self.assertEqual(observations[0]["instanceId"], INSTANCE) self.assertEqual(observations[0]["generation"], 1) self.assertEqual(observations[0]["state"], health if running and health != "healthy" else status) - self.assertEqual(observations[0]["healthy"], running and health == "healthy") + self.assertEqual(observations[0]["healthy"], status == "running" and running and health == "healthy") + self.docker.containers[self.agent.container_name(INSTANCE, 1)] = info + self.assertEqual(self.agent.inspect(INSTANCE, 1)["healthy"], observations[0]["healthy"]) self.assertIn("observedAt", observations[0]) self.assertEqual(sum(args.args[:2] == ("container", "inspect") for args in docker.call_args_list), 1) info["Mounts"][0]["Source"] = "/foreign/app" diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts index 714878e..b9d939f 100644 --- a/src/lib/server/hosts/index.ts +++ b/src/lib/server/hosts/index.ts @@ -255,14 +255,17 @@ export function validateInventory(value: unknown): HostInventory { } if (inventory.cpuUsagePercent !== undefined && (typeof inventory.cpuUsagePercent !== 'number' || !Number.isFinite(inventory.cpuUsagePercent) || inventory.cpuUsagePercent < 0 || inventory.cpuUsagePercent > 100)) throw new Error('Invalid inventory cpuUsagePercent.'); validateInstanceResources(inventory); - if (inventory.instanceStates !== undefined && (!Array.isArray(inventory.instanceStates) || inventory.instanceStates.length > 1000 - || inventory.instanceStates.some(state => !state || typeof state !== 'object' - || typeof state.instanceId !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(state.instanceId) - || !Number.isSafeInteger(state.generation) || state.generation < 1 - || typeof state.state !== 'string' || !/^[a-z-]{1,40}$/.test(state.state) - || typeof state.healthy !== 'boolean' || state.healthy && state.state !== 'running' - || typeof state.observedAt !== 'string' || state.observedAt.length > 40 || !Number.isFinite(Date.parse(state.observedAt))))) { - throw new Error('Invalid instance state inventory.'); + if (inventory.instanceStates !== undefined) { + if (!Array.isArray(inventory.instanceStates) || inventory.instanceStates.length > 1000) { + throw new Error('Invalid instance state inventory.'); + } + // A malformed server observation must not discard the host heartbeat. + inventory.instanceStates = inventory.instanceStates.filter(state => state && typeof state === 'object' + && typeof state.instanceId === 'string' && /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(state.instanceId) + && Number.isSafeInteger(state.generation) && state.generation >= 1 + && typeof state.state === 'string' && /^[a-z-]{1,40}$/.test(state.state) + && typeof state.healthy === 'boolean' && (!state.healthy || state.state === 'running') + && typeof state.observedAt === 'string' && state.observedAt.length <= 40 && Number.isFinite(Date.parse(state.observedAt))); } for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`); if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.'); diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py index f195d92..6297f00 100644 --- a/static/agent/ec-agent.py +++ b/static/agent/ec-agent.py @@ -32,7 +32,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.4.11" +VERSION = "1.4.12" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -1054,7 +1054,7 @@ class Agent: raise AgentError("Container ownership or generation does not match") state = info.get("State", {}) status = state.get("Status", "unknown") - healthy = state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy" + healthy = state.get("Status") == "running" and state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy" if status == "running" and not healthy: status = state.get("Health", {}).get("Status", "unknown") return {"exists": True, "running": state.get("Running") is True, "healthy": healthy, "state": status, "containerId": info.get("Id"), "instanceId": instance_id, "generation": generation, "startedAt": state.get("StartedAt"), "creationOperation": labels.get(LABEL + ".operation"), "specSha256": labels.get(LABEL + ".spec-sha256")} @@ -1152,7 +1152,7 @@ class Agent: if not any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", [])): continue state = info.get("State", {}) - healthy = state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy" + healthy = state.get("Status") == "running" and state.get("Running") is True and state.get("Health", {}).get("Status", "healthy") == "healthy" status = state.get("Status", "unknown") if status == "running" and not healthy: status = state.get("Health", {}).get("Status", "unknown") diff --git a/static/agent/manifest.json b/static/agent/manifest.json index 06e1487..4dda099 100644 --- a/static/agent/manifest.json +++ b/static/agent/manifest.json @@ -1,8 +1,8 @@ { - "version": "1.4.11", + "version": "1.4.12", "protocolVersion": 1, "platform": "debian-amd64", - "sha256": "4c6cc180f3c45a436878a0278d62b6840526150e039fdf51bafadef63759b131", + "sha256": "1e1bf2f04c02f0c8d0db8e4dbf39fadddc39c8bd04db5b8db753b6fe9e04c315", "updaterProtocolVersion": 1, "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8" } diff --git a/tests/unit/native-hosts.test.ts b/tests/unit/native-hosts.test.ts index 4290389..97c67c9 100644 --- a/tests/unit/native-hosts.test.ts +++ b/tests/unit/native-hosts.test.ts @@ -15,6 +15,19 @@ async function enrolled() { } describe('native host enrollment and scoped authentication', () => { + it('keeps the host online and delivers work when individual state observations are invalid', async () => { + const api = await enrolled(); + const good = { instanceId: 'healthy-server', generation: 1, state: 'running', healthy: true, observedAt: new Date().toISOString() }; + const operation = api.queueHostOperation(api.host.id, 'host.inspect', {}); + const host = api.heartbeat(api.host.id, { ...inventory, instanceStates: [good, + { ...good, instanceId: 'crashing', state: 'restarting' }, + { ...good, instanceId: 'paused', state: 'paused' }, null, + { ...good, generation: 0 }, { ...good, observedAt: 'invalid' }] }); + expect(host.inventory.instanceStates).toEqual([good]); + expect(host.readiness).toBe('ready'); + expect(api.pendingHostOperations(api.host.id).map(row => row.id)).toContain(operation.id); + }); + it('queues serialized launch files and processes receipts without hydrating plugin payloads', async () => { const api = await enrolled(); const { db } = await import('../../src/lib/server/db/index.js');