Keep agent lanes alive after controller response failures
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import base64
|
||||
import importlib.util
|
||||
import http.client
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
@@ -392,6 +393,59 @@ class AgentTests(unittest.TestCase):
|
||||
self.agent.journal.close()
|
||||
self.temp.cleanup()
|
||||
|
||||
def test_controller_truncated_response_is_a_sanitized_retryable_error(self):
|
||||
# Exercise Python's actual HTTP body reader against a truncated stream.
|
||||
sock = MagicMock()
|
||||
sock.makefile.return_value = io.BytesIO(b'HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{"secret":')
|
||||
response = http.client.HTTPResponse(sock)
|
||||
response.begin()
|
||||
with patch.object(module.urllib.request, "build_opener") as opener:
|
||||
opener.return_value.open.return_value = response
|
||||
with self.assertRaisesRegex(AgentError, "^Controller request failed$"):
|
||||
self.agent.request("GET", "/api/agent/operations")
|
||||
|
||||
def test_read_and_health_lanes_retry_unexpected_failures_without_logging_secrets(self):
|
||||
for lane in ("read", "health"):
|
||||
with self.subTest(lane=lane):
|
||||
self.agent.stopping.clear()
|
||||
polls = []
|
||||
|
||||
def request(*args):
|
||||
polls.append(args)
|
||||
if len(polls) == 1:
|
||||
raise RuntimeError("secret controller response")
|
||||
self.agent.stopping.set()
|
||||
return {"operations": []}
|
||||
|
||||
with patch.object(self.agent, "request", side_effect=request), patch.object(self.agent.stopping, "wait", return_value=False), patch.object(module.sys, "stderr", new_callable=io.StringIO) as logs:
|
||||
self.agent.read_loop(lane)
|
||||
self.assertEqual(len(polls), 2)
|
||||
self.assertIn("RuntimeError", logs.getvalue())
|
||||
self.assertNotIn("secret", logs.getvalue())
|
||||
|
||||
def test_heartbeat_loop_survives_unexpected_failures(self):
|
||||
with patch.object(self.agent.stopping, "wait", side_effect=[False, False, True]), patch.object(self.agent, "heartbeat", side_effect=[RuntimeError("secret"), None]) as heartbeat, patch.object(module.sys, "stderr", new_callable=io.StringIO) as logs:
|
||||
self.agent.heartbeat_loop()
|
||||
self.assertEqual(heartbeat.call_count, 2)
|
||||
self.assertIn("RuntimeError", logs.getvalue())
|
||||
self.assertNotIn("secret", logs.getvalue())
|
||||
|
||||
def test_startup_and_write_loop_survive_unexpected_failures(self):
|
||||
polls = []
|
||||
|
||||
def poll():
|
||||
polls.append(True)
|
||||
if len(polls) == 1:
|
||||
raise RuntimeError("secret")
|
||||
self.agent.stopping.set()
|
||||
|
||||
with patch.object(module.threading, "Thread"), patch.object(self.agent.stopping, "wait", return_value=False), patch.object(self.agent, "heartbeat", side_effect=[RuntimeError("secret"), None]) as heartbeat, patch.object(self.agent, "poll_operations", side_effect=poll), patch.object(module.sys, "stderr", new_callable=io.StringIO) as logs:
|
||||
self.agent.run()
|
||||
self.assertEqual(heartbeat.call_count, 2)
|
||||
self.assertEqual(len(polls), 2)
|
||||
self.assertEqual(logs.getvalue().count("RuntimeError"), 2)
|
||||
self.assertNotIn("secret", logs.getvalue())
|
||||
|
||||
def test_inventory_reuses_manifests_until_mtime_changes(self):
|
||||
marker = self.library / "manifest.json"
|
||||
saved = json.loads(marker.read_text())
|
||||
|
||||
Reference in New Issue
Block a user