Keep agent lanes alive after controller response failures

This commit is contained in:
2026-10-06 12:02:40 -04:00
parent 9bcdd8cc22
commit 532a10a880
4 changed files with 82 additions and 6 deletions
+54
View File
@@ -1,5 +1,6 @@
import base64
import importlib.util
import http.client
import io
import json
from pathlib import Path
@@ -392,6 +393,59 @@ class AgentTests(unittest.TestCase):
self.agent.journal.close()
self.temp.cleanup()
def test_controller_truncated_response_is_a_sanitized_retryable_error(self):
# Exercise Python's actual HTTP body reader against a truncated stream.
sock = MagicMock()
sock.makefile.return_value = io.BytesIO(b'HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{"secret":')
response = http.client.HTTPResponse(sock)
response.begin()
with patch.object(module.urllib.request, "build_opener") as opener:
opener.return_value.open.return_value = response
with self.assertRaisesRegex(AgentError, "^Controller request failed$"):
self.agent.request("GET", "/api/agent/operations")
def test_read_and_health_lanes_retry_unexpected_failures_without_logging_secrets(self):
for lane in ("read", "health"):
with self.subTest(lane=lane):
self.agent.stopping.clear()
polls = []
def request(*args):
polls.append(args)
if len(polls) == 1:
raise RuntimeError("secret controller response")
self.agent.stopping.set()
return {"operations": []}
with patch.object(self.agent, "request", side_effect=request), patch.object(self.agent.stopping, "wait", return_value=False), patch.object(module.sys, "stderr", new_callable=io.StringIO) as logs:
self.agent.read_loop(lane)
self.assertEqual(len(polls), 2)
self.assertIn("RuntimeError", logs.getvalue())
self.assertNotIn("secret", logs.getvalue())
def test_heartbeat_loop_survives_unexpected_failures(self):
with patch.object(self.agent.stopping, "wait", side_effect=[False, False, True]), patch.object(self.agent, "heartbeat", side_effect=[RuntimeError("secret"), None]) as heartbeat, patch.object(module.sys, "stderr", new_callable=io.StringIO) as logs:
self.agent.heartbeat_loop()
self.assertEqual(heartbeat.call_count, 2)
self.assertIn("RuntimeError", logs.getvalue())
self.assertNotIn("secret", logs.getvalue())
def test_startup_and_write_loop_survive_unexpected_failures(self):
polls = []
def poll():
polls.append(True)
if len(polls) == 1:
raise RuntimeError("secret")
self.agent.stopping.set()
with patch.object(module.threading, "Thread"), patch.object(self.agent.stopping, "wait", return_value=False), patch.object(self.agent, "heartbeat", side_effect=[RuntimeError("secret"), None]) as heartbeat, patch.object(self.agent, "poll_operations", side_effect=poll), patch.object(module.sys, "stderr", new_callable=io.StringIO) as logs:
self.agent.run()
self.assertEqual(heartbeat.call_count, 2)
self.assertEqual(len(polls), 2)
self.assertEqual(logs.getvalue().count("RuntimeError"), 2)
self.assertNotIn("secret", logs.getvalue())
def test_inventory_reuses_manifests_until_mtime_changes(self):
marker = self.library / "manifest.json"
saved = json.loads(marker.read_text())