Add persistent AFK reserves and recovery (Refs #151)

agent/tests/test_agent.py

@@ -1832,6 +1832,118 @@ class AgentTests(unittest.TestCase):
         self.assertNotIn(saved["moderation"]["token"], row["result"])
         self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
 
+    def afk_journal(self, game="t5", phase="active"):
+        path = self.agent.instance_dir(INSTANCE, 1) / f"app/runtime/tws-afk/{game}/scope.json"
+        reserve = {"action": "reserve", "server": "t5_zombie_test_1", "guid": "ace",
+                   "match": "match-one", "request_id": "reserve-one", "session_id": "session-one", "seconds": 1800}
+        module.atomic_json(path, {"version": 1, "server": reserve["server"], "endpoint": module.AFK_GAME_ENDPOINT,
+            "requests": [json.dumps(reserve)], "sessions": [{"body": json.dumps(reserve), "phase": phase,
+            "authorized_seconds": 0 if phase == "prepared" else 1800,
+            "consumed_seconds": 42 if phase == "active" else 0}]})
+        return path
+
+    def afk_acknowledgment(self, method, path, payload):
+        self.assertEqual((method, path), ("POST", "/api/agent/afk/settle"))
+        self.assertFalse(self.agent.inspect(INSTANCE, 1)["running"])
+        transfer = payload["transfer"]
+        self.assertEqual(transfer["action"], "settle")
+        return {"state": "unresolved" if transfer["consumed_seconds"] is None else "settled",
+                "request_id": transfer["request_id"]}
+
+    def test_afk_remove_recovers_both_engines_without_fabricating_crash_duration(self):
+        self.create()
+        paths = [self.afk_journal(game) for game in ("t5", "t6")]
+        def acknowledge(*args):
+            transfer = args[2]["transfer"]
+            self.assertTrue(all(path.exists() for path in paths))
+            self.assertEqual(transfer["end_reason"], "crash")
+            self.assertIsNone(transfer["consumed_seconds"])
+            self.assertEqual(transfer["confirmed_seconds"], 42)
+            self.assertRegex(transfer["request_id"], r"^afk-stop-[0-9a-f]{16}$")
+            return self.afk_acknowledgment(*args)
+        with patch.object(self.agent, "request", side_effect=acknowledge) as request:
+            result = self.agent.execute(self.operation("instance.remove", {"safetyOverride": True}, "remove"))
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(request.call_count, 2)
+        self.assertTrue(all(not path.exists() for path in paths))
+
+    def test_afk_recovery_retains_storage_and_reuses_decisions_after_lost_reply(self):
+        self.create()
+        path = self.afk_journal(phase="authorized")
+        operation = self.operation("instance.remove", {"safetyOverride": True}, "remove")
+        original = path.read_bytes()
+        observed = []
+        def lost(method, endpoint, payload):
+            observed.append(payload["transfer"])
+            raise AgentError("Lost reply")
+        with patch.object(self.agent, "request", side_effect=lost):
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+        self.assertEqual(path.read_bytes(), original)
+        self.agent.journal.close()
+        self.agent = Agent(self.config, self.docker)
+        def acknowledge(method, endpoint, payload):
+            self.assertEqual(payload["transfer"], observed[0])
+            self.assertEqual(payload["transfer"]["consumed_seconds"], 0)
+            self.assertEqual(payload["transfer"]["end_reason"], "never_started")
+            return self.afk_acknowledgment(method, endpoint, payload)
+        with patch.object(self.agent, "request", side_effect=acknowledge):
+            self.assertEqual(self.agent.execute(operation)["state"], "completed")
+        self.assertFalse(path.exists())
+
+    def test_afk_remove_accepts_an_already_corrected_session(self):
+        self.create()
+        path = self.afk_journal()
+        def acknowledge(method, endpoint, payload):
+            response = self.afk_acknowledgment(method, endpoint, payload)
+            return {**response, "state": "corrected"}
+        with patch.object(self.agent, "request", side_effect=acknowledge):
+            result = self.agent.execute(self.operation("instance.remove", {"safetyOverride": True}, "remove"))
+        self.assertEqual(result["state"], "completed", result)
+        self.assertFalse(path.exists())
+
+    def test_afk_recovery_rejects_running_process_corruption_and_untrusted_paths(self):
+        self.create()
+        path = self.afk_journal()
+        with self.assertRaisesRegex(AgentError, "stopped instance"):
+            self.agent.settle_afk(INSTANCE, 1, "remove")
+        operation = self.operation("instance.remove", {"safetyOverride": True}, "remove")
+        original = json.loads(path.read_text())
+        for invalid in ("{broken", json.dumps({**original, "endpoint": "https://untrusted.example"}),
+                        json.dumps({**original, "sessions": []}),
+                        json.dumps({**original, "requests": original["requests"] * 2})):
+            path.write_text(invalid)
+            with patch.object(self.agent, "request") as request:
+                self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+                request.assert_not_called()
+            self.assertTrue(path.exists())
+        outside = Path(self.temp.name) / "afk-outside.json"
+        outside.write_text(json.dumps(original))
+        path.unlink()
+        path.symlink_to(outside)
+        with patch.object(self.agent, "request") as request:
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+            request.assert_not_called()
+        self.assertTrue(outside.exists())
+
+    def test_afk_recovery_preserves_purchase_and_event_receipt_identity(self):
+        self.create()
+        path = self.afk_journal(phase="prepared")
+        value = json.loads(path.read_text())
+        for action in ("event", "purchase"):
+            value["requests"].append(json.dumps({"action": action, "server": value["server"], "guid": "ace",
+                "match": "match-one", "request_id": action + "-one", "confirmed": True}))
+        module.atomic_json(path, value)
+        expected = module.afk_recovery_requests(value)
+        def acknowledge(method, endpoint, payload):
+            self.assertEqual(endpoint, "/api/agent/afk/settle")
+            self.assertIn(payload["transfer"], expected)
+            return {"request_id": payload["transfer"]["request_id"],
+                    "state": "settled" if payload["action"] == "settle" else "rejected"}
+        with patch.object(self.agent, "request", side_effect=acknowledge) as request:
+            result = self.agent.execute(self.operation("instance.remove", {"safetyOverride": True}, "remove"))
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(request.call_count, 3)
+
     def bank_journal(self, decisions=("commit", "cancel"), game="t5", endpoint=module.BANK_GAME_ENDPOINT):
         path = self.agent.instance_dir(INSTANCE, 1) / f"app/runtime/tws-bank/{game}/{module.digest(endpoint.encode())}.json"
         reports = [json.dumps({"server": "t5_zombie_test_1", "guid": "ace", "match": "match-one", "request_id": f"{game}-{i}", "operation": "deposit", "amount": 2000, "capacity": 1000000, "decision": decision}) for i, decision in enumerate(decisions)]