josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Reuse heartbeat container state and batch host RCON status

Browse Source
This commit is contained in:
josh
2026-10-05 15:14:44 -04:00
parent 6586809b35
commit 239eceb779
12 changed files with 517 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/tests/test_agent.py
+82 -2
View File
@@ -472,7 +472,7 @@ class AgentTests(unittest.TestCase):
def resource_container(self):
return {"Name": self.agent.container_name(INSTANCE, 1),
"Config": {"Labels": {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1"}},
"State": {"Running": True, "StartedAt": "2000-01-01T00:00:00.000000000Z"},
"State": {"Running": True, "Status": "running", "StartedAt": "2000-01-01T00:00:00.000000000Z"},
"Mounts": [{"Type": "bind", "Source": str(self.agent.instance_dir(INSTANCE, 1) / "app"), "Destination": "/home/plutainer/app"}]}
def test_instance_resources_batches_owned_containers_and_normalizes_core_cpu_and_memory(self):
@@ -486,7 +486,7 @@ class AgentTests(unittest.TestCase):
self.assertEqual(samples[0]["cpuCores"], 1.5)
self.assertEqual(samples[0]["memoryBytes"], int(384.5 * 1024 ** 2))
self.assertGreater(samples[0]["uptimeSeconds"], 300)
self.assertEqual(call.call_args_list[0].args, ("container", "ls", "--filter", "label=io.electric-cherry.managed=true", "--no-trunc", "--format", "{{.ID}}"))
self.assertEqual(call.call_args_list[0].args, ("container", "ls", "--all", "--filter", "label=io.electric-cherry.managed=true", "--no-trunc", "--format", "{{.ID}}"))
self.assertEqual(call.call_args_list[-1].args, ("stats", "--no-stream", "--no-trunc", "--format", "{{json .}}", name))
def test_instance_resources_ignores_stopped_foreign_and_mismatched_containers(self):
@@ -520,6 +520,86 @@ class AgentTests(unittest.TestCase):
self.assertEqual(inventory["instanceResources"], [])
self.assertNotIn("instanceResourcesError", inventory)
def test_heartbeat_states_share_inspection_and_include_stopped_and_unhealthy_containers(self):
for status, running, health in (("exited", False, "healthy"), ("running", True, "unhealthy"), ("running", True, "healthy")):
info = self.resource_container()
info["State"].update({"Status": status, "Running": running, "Health": {"Status": health}})
observations = []
outputs = ["container-id\n", json.dumps([info])] + ([""] if running else [])
with patch.object(self.docker, "call", side_effect=outputs) as docker:
self.agent.instance_resources(observations)
self.assertEqual(observations[0]["instanceId"], INSTANCE)
self.assertEqual(observations[0]["generation"], 1)
self.assertEqual(observations[0]["state"], health if running and health != "healthy" else status)
self.assertEqual(observations[0]["healthy"], running and health == "healthy")
self.assertIn("observedAt", observations[0])
self.assertEqual(sum(args.args[:2] == ("container", "inspect") for args in docker.call_args_list), 1)
info["Mounts"][0]["Source"] = "/foreign/app"
observations = []
with patch.object(self.docker, "call", side_effect=["container-id", json.dumps([info])]):
self.agent.instance_resources(observations)
self.assertEqual(observations, [])
def test_status_batch_limits_concurrency_and_isolates_target_failures(self):
operation = {"kind": "rcon.status-batch", "payload": {"instances": [
{"instanceId": f"00000000-0000-4000-8000-{i:012d}", "generation": 1} for i in range(5)]}}
self.assertTrue(module.read_operation(operation))
self.assertTrue(self.agent.health_operation(operation))
ready, release = threading.Event(), threading.Event()
lock = threading.Lock()
active, peak = 0, 0
def rcon(instance_id, generation, command, deadline):
nonlocal active, peak
with lock:
active += 1
peak = max(peak, active)
if active == 2:
ready.set()
try:
self.assertEqual(command, "status")
self.assertTrue(release.wait(2))
if instance_id == "00000000-0000-4000-8000-000000000002":
raise AgentError("local failure")
return {"ok": True, "text": "status"}
finally:
with lock:
active -= 1
results = []
with patch.object(self.agent, "rcon", side_effect=rcon) as probe:
thread = threading.Thread(target=lambda: results.append(self.agent.dispatch(operation)))
thread.start()
try:
self.assertTrue(ready.wait(2))
self.assertEqual(probe.call_count, 2)
finally:
release.set()
thread.join(timeout=3)
self.assertFalse(thread.is_alive())
self.assertEqual(peak, 2)
self.assertEqual(len(results[0]["results"]), 5)
self.assertFalse(results[0]["ok"])
self.assertEqual(results[0]["results"][2]["reply"]["reason"], "unreachable")
self.assertTrue(results[0]["results"][4]["reply"]["ok"])
def test_status_batch_rejects_old_generations_and_bounds_expired_work(self):
marker = self.agent.instance_dir(INSTANCE, 1).parent / "current.json"
module.atomic_json(marker, {"generation": 2})
operation = {"kind": "rcon.status-batch", "payload": {"instances": [{"instanceId": INSTANCE, "generation": 1}]}}
with patch.object(self.agent, "rcon") as probe:
result = self.agent.dispatch(operation)
self.assertEqual(result["results"][0]["reply"]["reason"], "superseded")
probe.assert_not_called()
marker.unlink()
with patch.object(module.time, "monotonic", side_effect=[0] + [21] * 20):
result = self.agent.dispatch(operation)
self.assertEqual(result["results"][0]["reply"]["reason"], "timeout")
probe.assert_not_called()
for targets in ([], [None], [{"instanceId": INSTANCE, "generation": 1}] * 2):
with self.assertRaises(AgentError):
self.agent.dispatch({"kind": "rcon.status-batch", "payload": {"instances": targets}})
def operation(self, kind, payload=None, op_id="operation-1", generation=1):
return {"id": op_id, "kind": kind, "instanceId": INSTANCE, "generation": generation, "payload": payload or {}}