Report stalled agent lanes and bound health lock waits

agent/tests/test_agent.py

@@ -446,6 +446,67 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(logs.getvalue().count("RuntimeError"), 2)
         self.assertNotIn("secret", logs.getvalue())
 
+    def test_lane_inventory_tracks_each_health_worker_and_detects_dead_threads(self):
+        def poll():
+            self.agent.lane_started("health-1")
+            self.agent.lane_polled("health-1")
+
+        with patch.object(module.time, "monotonic", return_value=100):
+            thread = threading.Thread(target=poll)
+            thread.start()
+            thread.join(timeout=2)
+        self.assertFalse(thread.is_alive())
+        with patch.object(module.time, "monotonic", return_value=150):
+            self.agent.lane_started("health-2")
+            self.agent.lane_polled("health-2")
+            self.agent.lane_busy("health-2", True)
+        with patch.object(module.time, "monotonic", return_value=200):
+            lanes = {lane["name"]: lane for lane in self.agent.lane_inventory()}
+        self.assertEqual(set(lanes), {"health-1", "health-2", "read", "write"})
+        self.assertFalse(lanes["health-1"]["alive"])
+        self.assertEqual(lanes["health-1"]["pollAgeSeconds"], 100)
+        self.assertTrue(lanes["health-2"]["alive"])
+        self.assertTrue(lanes["health-2"]["busy"])
+        self.assertEqual(lanes["health-2"]["pollAgeSeconds"], 50)
+        self.assertIsNotNone(lanes["health-2"]["lastPollAt"])
+        self.assertEqual(len(self.agent.inventory()["agentLanes"]), 4)
+
+    def test_health_lane_times_out_on_instance_lock_and_processes_next_check(self):
+        blocked = self.operation("instance.inspect", op_id="blocked-health")
+        fresh = {**self.operation("instance.inspect", op_id="fresh-health"), "instanceId": "other-instance"}
+        pending = iter([blocked, fresh])
+        failures = []
+
+        def request(method, path, payload=None):
+            if method == "POST":
+                return {"operation": {"id": path.rsplit("/", 1)[1], "state": payload["state"]}}
+            operation = next(pending, None)
+            if operation is None:
+                self.agent.stopping.set()
+            return {"operations": [operation] if operation else []}
+
+        def read():
+            try:
+                self.agent.read_loop("health")
+            except BaseException as error:
+                failures.append(error)
+
+        with patch.object(module, "HEALTH_LOCK_TIMEOUT_SECONDS", 0.01), patch.object(self.agent, "request", side_effect=request), patch.object(self.agent, "dispatch", return_value={"healthy": True}) as dispatch:
+            with self.agent.instance_lock(blocked):
+                thread = threading.Thread(target=read)
+                thread.start()
+                thread.join(timeout=2)
+                finished_while_locked = not thread.is_alive()
+            self.agent.stopping.set()
+            thread.join(timeout=2)
+        self.assertTrue(finished_while_locked)
+        self.assertEqual(failures, [])
+        self.assertEqual(self.agent.journal.get(blocked["id"])["state"], "failed")
+        self.assertIn("waiting for the instance", self.agent.journal.get(blocked["id"])["error"])
+        self.assertEqual(self.agent.journal.get(fresh["id"])["state"], "completed")
+        dispatch.assert_called_once_with(fresh)
+        self.assertEqual(self.agent.read_claims, set())
+
     def test_inventory_reuses_manifests_until_mtime_changes(self):
         marker = self.library / "manifest.json"
         saved = json.loads(marker.read_text())