Fix native health polling and isolate file operation traffic

agent/tests/test_agent.py

@@ -6,7 +6,7 @@ from pathlib import Path
 import tempfile
 import threading
 import unittest
-from unittest.mock import mock_open, patch
+from unittest.mock import MagicMock, call, mock_open, patch
 
 spec = importlib.util.spec_from_file_location("ec_agent", Path(__file__).resolve().parents[1] / "ec_agent.py")
 module = importlib.util.module_from_spec(spec)
@@ -995,6 +995,64 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(result["players"], 1)
         self.assertTrue(result["ok"])
 
+    @staticmethod
+    def rcon_socket(*replies):
+        sock = MagicMock()
+        sock.__enter__.return_value = sock
+        sock.recv.side_effect = replies
+        return sock
+
+    def test_udp_status_recovers_after_transient_failures(self):
+        self.create()
+        for error in (module.socket.timeout(), OSError("unreachable")):
+            with self.subTest(error=type(error).__name__):
+                failed = self.rcon_socket(error)
+                # Keep all response packets, including occupied player rows.
+                recovered = self.rcon_socket(
+                    b"\xff\xff\xff\xffprint\nmap: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n",
+                    b"\xff\xff\xff\xffprint\n0 42 CNCT abc player\n", module.socket.timeout())
+                with patch.object(module.socket, "socket", side_effect=[failed, recovered]) as sockets, patch.object(module.time, "sleep") as sleep:
+                    result = self.agent.rcon(INSTANCE, 1, "status")
+                self.assertTrue(result["ok"])
+                self.assertEqual(result["players"], 1)
+                self.assertEqual(sockets.call_count, 2)
+                sleep.assert_called_once_with(0.25)
+                failed.__exit__.assert_called_once()
+                recovered.__exit__.assert_called_once()
+
+    def test_udp_status_retries_are_bounded(self):
+        self.create()
+        attempts = [self.rcon_socket(module.socket.timeout()) for _ in range(3)]
+        with patch.object(module.socket, "socket", side_effect=attempts) as sockets, patch.object(module.time, "sleep") as sleep:
+            result = self.agent.rcon(INSTANCE, 1, "status")
+        self.assertFalse(result["ok"])
+        self.assertEqual(result["reason"], "timeout")
+        self.assertEqual(sockets.call_count, 3)
+        self.assertEqual(sleep.call_args_list, [call(0.25), call(0.5)])
+
+    def test_udp_status_does_not_retry_bad_replies_or_partial_tables(self):
+        self.create()
+        for packet in (b"wrong listener", b"\xff\xff\xff\xffprint\nBad rcon_password",
+                       b"\xff\xff\xff\xffprint\nmap: mp_kino\nnum score ping guid name\n"):
+            with self.subTest(packet=packet):
+                sock = self.rcon_socket(packet, module.socket.timeout())
+                with patch.object(module.socket, "socket", return_value=sock) as sockets, patch.object(module.time, "sleep") as sleep:
+                    result = self.agent.rcon(INSTANCE, 1, "status")
+                self.assertIsNone(result["players"])
+                sockets.assert_called_once()
+                sleep.assert_not_called()
+
+    def test_udp_commands_with_effects_are_never_replayed(self):
+        self.create()
+        for command in ("map_restart", "kick 0", "version"):
+            with self.subTest(command=command):
+                sock = self.rcon_socket(module.socket.timeout())
+                with patch.object(module.socket, "socket", return_value=sock) as sockets, patch.object(module.time, "sleep") as sleep:
+                    result = self.agent.rcon(INSTANCE, 1, command)
+                self.assertEqual(result["reason"], "timeout")
+                sockets.assert_called_once()
+                sleep.assert_not_called()
+
     def test_t6_bot_column_allows_empty_restart_but_holds_players_and_partial_replies(self):
         self.create()
         header = "map: zm_nuked\nnum score bot ping guid                             name             lastmsg address               qport  rate\n"

agent/tests/test_update_integration.py

@@ -118,6 +118,44 @@ class UpdateIntegrationTests(unittest.TestCase):
             self.agent.read_loop()
             dispatch.assert_not_called()
 
+    def test_health_lane_runs_while_file_receipt_for_same_instance_is_slow(self):
+        receipt_waiting, release_receipt, health_done = threading.Event(), threading.Event(), threading.Event()
+        file_op = {"id": "file-one", "kind": "files.read", "instanceId": "same", "payload": {}}
+        health_op = {"id": "health-one", "kind": "instance.inspect", "instanceId": "same", "payload": {}}
+        errors = []
+        def request(method, path, body=None):
+            if method == "GET":
+                return {"operations": [health_op if "lane=health" in path else file_op]}
+            if path.endswith("file-one") and body["state"] == "completed":
+                receipt_waiting.set()
+                release_receipt.wait(3)
+            return {}
+        def dispatch(operation):
+            if operation["id"] == "health-one":
+                health_done.set()
+                self.agent.stopping.set()
+            return {"ok": True}
+        def run(lane):
+            try: self.agent.read_loop(lane)
+            except Exception as error: errors.append(error)
+        self.agent.request, self.agent.dispatch = request, dispatch
+        files = threading.Thread(target=run, args=("read",))
+        health = threading.Thread(target=run, args=("health",))
+        files.start()
+        try:
+            self.assertTrue(receipt_waiting.wait(2))
+            health.start()
+            self.assertTrue(health_done.wait(2), "Health waited for a file receipt")
+        finally:
+            self.agent.stopping.set()
+            release_receipt.set()
+            files.join(3)
+            if health.ident is not None: health.join(3)
+        self.assertFalse(files.is_alive())
+        self.assertFalse(health.is_alive())
+        self.assertEqual(errors, [])
+        self.assertEqual(self.agent.journal.get("health-one")["state"], "completed")
+
     def test_instance_mutations_and_global_mutations_exclude_observations(self):
         operation = {"instanceId": "one"}
         entered = threading.Event()