Improve application responsiveness and background processing

agent/tests/test_update_integration.py

@@ -4,6 +4,7 @@ import json
 import os
 from pathlib import Path
 import tempfile
+import threading
 import unittest
 from unittest.mock import patch
 
@@ -43,11 +44,15 @@ class UpdateIntegrationTests(unittest.TestCase):
             with self.assertRaises(BlockingIOError):
                 fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
 
-    def test_operation_delivery_execution_and_reports_exclude_updates(self):
-        operation = {"id": "one", "kind": "host.inspect", "payload": {}}
+    def test_execution_and_reports_exclude_updates_but_long_poll_does_not(self):
+        operation = {"id": "one", "kind": "runtime.pull", "payload": {}}
         calls = []
         def request(method, path, body=None):
-            self.assert_activity_held()
+            if method != "GET":
+                self.assert_activity_held()
+            else:
+                with (self.agent.root / "agent" / "activity.lock").open("a") as stream:
+                    fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
             calls.append((method, path))
             return {"operations": [operation]} if method == "GET" else {}
         def dispatch(value):
@@ -82,7 +87,61 @@ class UpdateIntegrationTests(unittest.TestCase):
         module.UPDATE_JOURNAL.unlink()
         with patch.object(self.agent, "request", return_value={"operations": []}) as request:
             self.agent.poll_operations()
-            request.assert_called_once_with("GET", "/api/agent/operations")
+            request.assert_called_once_with("GET", "/api/agent/operations?lane=write&wait=20")
+
+    def test_read_lane_completes_while_mutation_is_busy_and_excludes_updater(self):
+        read_done = threading.Event()
+        operation = {"id": "read-one", "kind": "host.inspect", "payload": {}}
+        self.agent.request = lambda method, path, body=None: {"operations": [operation]} if method == "GET" else {}
+        def dispatch(value):
+            self.assert_activity_held()
+            read_done.set()
+            self.agent.stopping.set()
+            return {"ok": True}
+        self.agent.dispatch = dispatch
+        with self.agent.activity() as acquired:
+            self.assertTrue(acquired)
+            reader = threading.Thread(target=self.agent.read_loop)
+            reader.start()
+            self.assertTrue(read_done.wait(2), "Read waited behind unrelated mutation activity")
+            reader.join(2)
+        self.assertFalse(reader.is_alive())
+        self.assertEqual(self.agent.journal.get("read-one")["state"], "completed")
+        self.assertIsNone(self.agent.executing_id)
+
+    def test_read_lane_never_dispatches_mutations(self):
+        def request(method, path, body=None):
+            self.agent.stopping.set()
+            return {"operations": [{"id": "bad-read", "kind": "instance.remove", "payload": {}}]}
+        self.agent.request = request
+        with patch.object(self.agent, "dispatch") as dispatch:
+            self.agent.read_loop()
+            dispatch.assert_not_called()
+
+    def test_instance_mutations_and_global_mutations_exclude_observations(self):
+        operation = {"instanceId": "one"}
+        entered = threading.Event()
+        release = threading.Event()
+        def read():
+            with self.agent.read_gate.read(), self.agent.instance_lock(operation):
+                entered.set()
+                release.wait(2)
+        with self.agent.instance_lock(operation):
+            reader = threading.Thread(target=read)
+            reader.start()
+            self.assertFalse(entered.wait(0.05))
+        self.assertTrue(entered.wait(2))
+        writer_entered = threading.Event()
+        def write():
+            with self.agent.read_gate.write():
+                writer_entered.set()
+        writer = threading.Thread(target=write)
+        writer.start()
+        self.assertFalse(writer_entered.wait(0.05))
+        release.set()
+        reader.join(2)
+        writer.join(2)
+        self.assertTrue(writer_entered.is_set())
 
     def test_failed_controller_contact_never_writes_healthy_receipt(self):
         with patch.object(self.agent, "request", side_effect=module.AgentError("offline")):