Read chat logs incrementally and drain retention (Refs #62)

agent/tests/test_agent.py

@@ -3,6 +3,7 @@ import importlib.util
 import http.client
 import io
 import json
+import os
 from pathlib import Path
 import tempfile
 import threading
@@ -2699,6 +2700,71 @@ class AgentTests(unittest.TestCase):
         self.agent.rcon = lambda *_: {"ok": True, "players": 0}
         self.assertEqual(self.agent.execute(operation)["state"], "completed")
 
+    def test_incremental_log_reads_append_bytes_without_rereading_unchanged_content(self):
+        self.create()
+        app = self.agent.instance_dir(INSTANCE, 1) / "app"
+        (app / "logs").mkdir()
+        path = app / "logs/games_sp.log"
+        path.write_bytes(b"first\n")
+        payload = {"path": "logs/games_sp.log", "mode": "incremental"}
+        result = self.agent.execute(self.operation("logs.read", payload))
+        self.assertEqual(result["state"], "completed")
+        first = json.loads(result["result"])
+        self.assertEqual(base64.b64decode(first["contentBase64"]), b"first\n")
+        self.assertTrue(first["reset"])
+        # An unchanged stat must not read even the cursor anchor.
+        with path.open("rb") as stream, patch.object(stream, "read", side_effect=AssertionError("unchanged file was read")):
+            unchanged = self.agent.read_log_chunk(stream, os.fstat(stream.fileno()), first["cursor"])
+        self.assertEqual(unchanged["contentBase64"], "")
+        with path.open("ab") as stream:
+            stream.write("café\n".encode())
+        appended = self.agent.read_log(INSTANCE, 1, {**payload, "cursor": first["cursor"]})
+        self.assertEqual(base64.b64decode(appended["contentBase64"]), "café\n".encode())
+        self.assertFalse(appended["reset"])
+        self.assertEqual(appended["cursor"]["offset"], len("first\ncafé\n".encode()))
+        self.assertIn("logs.read-offset", self.agent.inventory()["capabilities"])
+
+    def test_incremental_log_rotation_and_copy_truncate_reset_the_cursor(self):
+        self.create()
+        app = self.agent.instance_dir(INSTANCE, 1) / "app"
+        (app / "logs").mkdir()
+        path = app / "logs/games_sp.log"
+        header = b"same game header\n" * 30
+        path.write_bytes(header + b"original\n")
+        payload = {"path": "logs/games_sp.log", "mode": "incremental"}
+        result = self.agent.read_log(INSTANCE, 1, payload)
+        path.rename(path.with_suffix(".old"))
+        for content in (header + b"rotated!\n", header + b"rewritten and longer\n", b"short\n", b""):
+            with self.subTest(content=content):
+                path.write_bytes(content)
+                result = self.agent.read_log(INSTANCE, 1, {**payload, "cursor": result["cursor"]})
+                self.assertTrue(result["reset"])
+                self.assertEqual(base64.b64decode(result["contentBase64"]), content)
+
+    def test_incremental_log_backlogs_are_bounded_and_preserve_split_utf8_bytes(self):
+        self.create()
+        app = self.agent.instance_dir(INSTANCE, 1) / "app"
+        (app / "logs").mkdir()
+        path = app / "logs/games_sp.log"
+        window = 256 * 1024
+        path.write_bytes(b"x" * (window + 10))
+        payload = {"path": "logs/games_sp.log", "mode": "incremental"}
+        initial = self.agent.read_log(INSTANCE, 1, payload)
+        self.assertTrue(initial["truncated"])
+        self.assertEqual(len(base64.b64decode(initial["contentBase64"])), window)
+        data = b"y" * (window - 1) + "é\n".encode()
+        with path.open("ab") as stream:
+            stream.write(data)
+        first = self.agent.read_log(INSTANCE, 1, {**payload, "cursor": initial["cursor"]})
+        second = self.agent.read_log(INSTANCE, 1, {**payload, "cursor": first["cursor"]})
+        self.assertFalse(first["reset"])
+        self.assertEqual(len(base64.b64decode(first["contentBase64"])), window)
+        self.assertEqual(base64.b64decode(first["contentBase64"]) + base64.b64decode(second["contentBase64"]), data)
+        self.assertEqual(second["cursor"]["offset"], second["size"])
+        for cursor in ({}, {**second["cursor"], "offset": -1}, {**second["cursor"], "anchorLength": 257}, {**second["cursor"], "offset": True}):
+            with self.subTest(cursor=cursor), self.assertRaisesRegex(AgentError, "Invalid log cursor"):
+                self.agent.read_log(INSTANCE, 1, {**payload, "cursor": cursor})
+
     def test_game_log_can_follow_internal_relative_plutainer_symlink(self):
         self.create()
         app = self.agent.instance_dir(INSTANCE, 1) / "app"
@@ -2709,6 +2775,8 @@ class AgentTests(unittest.TestCase):
         result = self.agent.execute(self.operation("logs.read", {"path": "logs/games_sp.log", "lines": 2}))
         self.assertEqual(result["state"], "completed")
         self.assertEqual(json.loads(result["result"])["lines"], ["second", "third"])
+        incremental = self.agent.read_log(INSTANCE, 1, {"path": "logs/games_sp.log", "mode": "incremental"})
+        self.assertEqual(base64.b64decode(incremental["contentBase64"]), b"first\nsecond\nthird\n")
         with self.assertRaises(AgentError):
             module.managed_write(app, "logs/games_sp.log", b"must-not-write-through-link")
 
@@ -2722,6 +2790,8 @@ class AgentTests(unittest.TestCase):
         result = self.agent.execute(self.operation("logs.read", {"path": "logs/games_sp.log"}))
         self.assertEqual(result["state"], "failed")
         self.assertIn("outside", result["error"])
+        with self.assertRaisesRegex(AgentError, "outside"):
+            self.agent.read_log(INSTANCE, 1, {"path": "logs/games_sp.log", "mode": "incremental"})
 
     def test_console_log_falls_back_to_owned_docker_output_including_stderr(self):
         self.create()