josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add configurable retention for successful routine operations

Browse Source
This commit is contained in:
josh
2026-10-05 13:09:44 -04:00
parent 2d069907f2
commit e6912ceb57
14 changed files with 590 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/tests/test_agent.py
+188
View File
@@ -57,6 +57,115 @@ class Response(io.BytesIO):
self.headers = headers or {}
class JournalRetentionTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.path = Path(self.temp.name) / "operations.sqlite3"
self.journal = Journal(self.path)
self.now = 2000000
self.sequence = 0
def tearDown(self):
self.journal.close()
self.temp.cleanup()
def record(self, kind="files.stat", payload=None, result=None, state="completed", error=None, at=None, confirmed=True):
self.sequence += 1
operation = {"id": f"operation-{self.sequence}", "kind": kind, "payload": payload or {}}
self.journal.accept(operation)
with patch.object(module.time, "time", return_value=self.now - 172800 if at is None else at):
row = self.journal.update(operation["id"], state, result, error)
if confirmed:
self.journal.confirm_completed(row)
return row
def prune(self, days=1):
with patch.object(module.time, "time", return_value=self.now):
return self.journal.prune_observations(days)
def test_cutoff_requires_an_older_confirmed_success(self):
old = self.record(at=self.now - 86401)
boundary = self.record(at=self.now - 86400)
recent = self.record(at=self.now - 10)
unconfirmed = self.record(confirmed=False)
self.assertEqual(self.prune(), 1)
with self.assertRaisesRegex(AgentError, "Unknown operation"):
self.journal.get(old["id"])
for row in (boundary, recent, unconfirmed):
self.assertEqual(self.journal.get(row["id"])["state"], "completed")
def test_zero_and_invalid_retention_preserve_every_record(self):
row = self.record()
for days in (0, -1, 36501, True, False, None, "1", 1.0, 1.5):
with self.subTest(days=days):
self.assertEqual(self.prune(days), 0)
self.assertEqual(self.journal.get(row["id"])["state"], "completed")
def test_all_read_kinds_are_eligible_but_mutations_keep_their_replay_journal(self):
for kind in ("host.inspect", "instance.inspect", "files.read", "files.list", "files.stat", "files.manifest", "logs.read"):
self.record(kind, result={"healthy": True})
for command in ("status", "getstatus", "version"):
self.record("rcon.execute", {"command": command}, {"ok": True})
mutations = [self.record(kind) for kind in ("instance.restart", "instance.stop", "instance.create", "files.stage", "files.activate", "credentials.rotate")]
mutations.append(self.record("rcon.execute", {"command": "map_restart"}, {"ok": True}))
self.assertEqual(self.prune(), 10)
for row in mutations:
saved, accepted = self.journal.accept(json.loads(row["request"]))
self.assertFalse(accepted)
self.assertEqual(saved["id"], row["id"])
def test_errors_failures_and_unfinished_operations_are_retained(self):
retained = [self.record(result=result) for result in ({"ok": False}, {"healthy": False}, {"error": "partial result"}, {"error": False}, {"error": []})]
retained.append(self.record(error="read did not finish"))
retained.extend(self.record(state=state) for state in ("accepted", "running", "unknown", "failed"))
self.assertEqual(self.prune(), 0)
for row in retained:
self.assertEqual(self.journal.get(row["id"])["state"], row["state"])
def test_old_receipt_cannot_confirm_a_changed_or_reexecuted_result(self):
old = self.record(confirmed=False)
with patch.object(module.time, "time", return_value=self.now - 90000):
current = self.journal.update(old["id"], "completed", {"new": "result"})
self.journal.confirm_completed(old)
self.assertIsNone(self.journal.get(old["id"])["reported_at"])
self.assertEqual(self.prune(), 0)
self.journal.confirm_completed(current)
self.assertEqual(self.prune(), 1)
def test_pruning_uses_bounded_indexed_batches(self):
template = self.record()
self.journal.db.executemany("""INSERT INTO operations(id,fingerprint,request,state,result,updated,reported_at,retention_eligible)
VALUES(?,?,?,'completed',NULL,?,?,1)""", [
(f"batch-{index}", template["fingerprint"], template["request"], self.now - 172800, self.now - 172800)
for index in range(504)
])
self.journal.db.commit()
self.assertEqual(self.prune(), 500)
self.assertEqual(self.journal.db.execute("SELECT COUNT(*) FROM operations").fetchone()[0], 5)
self.assertEqual(self.prune(), 5)
plan = self.journal.db.execute("""EXPLAIN QUERY PLAN SELECT id FROM operations
WHERE state='completed' AND reported_at IS NOT NULL AND retention_eligible=1 AND updated<? ORDER BY updated,id LIMIT 500""", (self.now,)).fetchall()
self.assertIn("operations_observation_retention", " ".join(str(tuple(row)) for row in plan))
def test_migration_keeps_legacy_completions_until_their_receipt_is_reacknowledged(self):
self.journal.close()
self.path.unlink()
legacy = module.sqlite3.connect(self.path)
legacy.execute("CREATE TABLE operations(id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, request TEXT NOT NULL, state TEXT NOT NULL, result TEXT, error TEXT, updated REAL NOT NULL)")
request = {"id": "legacy-read", "kind": "files.list", "instanceId": None, "generation": None, "payload": {}}
canonical = json.dumps(request, sort_keys=True, separators=(",", ":"))
legacy.execute("INSERT INTO operations VALUES(?,?,?,'completed',?,NULL,?)", (request["id"], module.digest(canonical.encode()), canonical, '{"entries":[]}', self.now - 172800))
legacy.commit()
legacy.close()
self.journal = Journal(self.path)
row, accepted = self.journal.accept(request)
self.assertFalse(accepted)
self.assertIsNone(row["reported_at"])
self.assertEqual(self.prune(), 0)
self.journal.confirm_completed(row)
self.assertEqual(self.prune(), 1)
class ManifestTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
@@ -1418,6 +1527,85 @@ class AgentTests(unittest.TestCase):
self.agent.heartbeat()
self.assertEqual(self.agent.config["portRange"], {"min": 30000, "max": 30100})
def test_heartbeat_requires_valid_retention_and_never_changes_the_journal(self):
self.assertEqual(self.agent.observation_keep_days, 0)
with patch.object(self.agent.journal, "prune_observations", side_effect=AssertionError("heartbeat must not use the journal")), patch.object(self.agent.journal, "confirm_completed", side_effect=AssertionError("heartbeat must not use the journal")):
for value, expected in ((1, 1), (36500, 36500), (0, 0), (True, 0), (False, 0), (1.0, 0), ("1", 0), (-1, 0), (36501, 0), (None, 0)):
with self.subTest(value=value):
self.agent.observation_keep_days = 3
self.agent.request = lambda *_, value=value: {"observationKeepDays": value}
self.agent.heartbeat()
self.assertEqual(self.agent.observation_keep_days, expected)
self.agent.observation_keep_days = 3
self.agent.request = lambda *_: {}
self.agent.heartbeat()
self.assertEqual(self.agent.observation_keep_days, 0)
def test_lost_completed_receipt_is_retained_and_replay_reports_the_saved_result(self):
operation = self.operation("files.stat", {"path": "configs/server.cfg"})
self.agent.journal.accept(operation)
with patch.object(module.time, "time", return_value=module.time.time() - 172800):
row = self.agent.journal.update(operation["id"], "completed", {"size": 8})
self.agent.request = MagicMock(side_effect=AgentError("receipt lost"))
with self.assertRaisesRegex(AgentError, "receipt lost"):
self.agent.report(row)
self.assertIsNone(self.agent.journal.get(operation["id"])["reported_at"])
self.assertEqual(self.agent.journal.prune_observations(1), 0)
with patch.object(self.agent, "dispatch", side_effect=AssertionError("completed read must not be executed again")):
replay = self.agent.execute(operation)
self.assertEqual(replay["result"], row["result"])
for response in ({}, None, {"operation": {"id": "different", "state": "completed"}}, {"operation": {"id": operation["id"], "state": "running"}}):
self.agent.request = lambda *_, response=response: response
self.agent.report(replay)
self.assertIsNone(self.agent.journal.get(operation["id"])["reported_at"])
self.agent.request = lambda *_: {"operation": {"id": operation["id"], "state": "completed"}}
self.agent.report(replay)
self.assertIsNotNone(self.agent.journal.get(operation["id"])["reported_at"])
self.assertEqual(self.agent.journal.prune_observations(1), 1)
def test_threaded_read_lane_confirms_and_prunes_through_its_own_connection(self):
self.create()
old_operation = self.operation("files.list", op_id="old-observation")
self.agent.journal.accept(old_operation)
with patch.object(module.time, "time", return_value=module.time.time() - 172800):
old = self.agent.journal.update(old_operation["id"], "completed", {"entries": []})
self.agent.journal.confirm_completed(old)
self.agent.observation_keep_days = 1
operation = self.operation("files.stat", {"path": "configs/server.cfg"}, "thread-observation")
polls = 0
failures = []
def request(method, path, payload=None):
nonlocal polls
if method == "POST":
return {"operation": {"id": operation["id"], "state": payload["state"]}}
polls += 1
if polls == 1:
return {"operations": [operation]}
self.agent.stopping.set()
return {"operations": []}
def run():
try:
self.agent.read_loop()
except BaseException as error:
failures.append(error)
self.agent.request = request
with patch.object(self.agent.journal, "confirm_completed", side_effect=AssertionError("main journal used by reader")), patch.object(self.agent.journal, "prune_observations", side_effect=AssertionError("main journal used by reader")):
thread = threading.Thread(target=run)
thread.start()
thread.join(timeout=5)
self.agent.stopping.set()
thread.join(timeout=5)
self.assertFalse(thread.is_alive())
self.assertEqual(failures, [])
self.assertEqual(polls, 2)
self.assertIsNotNone(self.agent.journal.get(operation["id"])["reported_at"])
with self.assertRaisesRegex(AgentError, "Unknown operation"):
self.agent.journal.get(old_operation["id"])
self.assertEqual(self.agent.journal.get("create")["state"], "completed")
def test_download_content_permissions_allow_container_read_access_with_private_umask(self):
content = b"complete-game-file"
manifest = self.manifest(content)