Report safe diagnostics for unexpected agent failures
This commit is contained in:
@@ -404,6 +404,128 @@ class AgentTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(AgentError, "^Controller request failed$"):
|
||||
self.agent.request("GET", "/api/agent/operations")
|
||||
|
||||
def test_unexpected_create_http_failure_reports_safe_diagnostics_and_does_not_replay(self):
|
||||
secret = "private-token-must-not-appear"
|
||||
error = module.urllib.error.HTTPError(f"https://user:{secret}@controller.example/blobs?token={secret}", 404,
|
||||
secret, {"Authorization": "Bearer " + secret}, None)
|
||||
error.__cause__ = RuntimeError(secret)
|
||||
payload = {"game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest",
|
||||
"port": 28960, "rconPassword": secret, "files": [{"path": "configs/server.cfg", "contentSha256": "a" * 64}]}
|
||||
operation = self.operation("instance.create", payload)
|
||||
with patch.object(module.urllib.request, "build_opener") as opener:
|
||||
opener.return_value.open.side_effect = error
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIsNone(result["checkpoint"])
|
||||
self.assertEqual(self.agent.execute(operation), result)
|
||||
opener.return_value.open.assert_called_once()
|
||||
details = json.loads(result["result"])["unexpectedError"]
|
||||
self.assertEqual(details["type"], "HTTPError")
|
||||
self.assertEqual(details["httpStatus"], 404)
|
||||
self.assertEqual(details["phase"], "validating-create-spec")
|
||||
self.assertEqual(details["agentVersion"], module.VERSION)
|
||||
self.assertEqual(details["frames"][-1]["function"], "artifact_open")
|
||||
self.assertLessEqual(len(details["frames"]), 8)
|
||||
self.assertTrue(all(set(frame) == {"function", "line"} and frame["line"] > 0 for frame in details["frames"]))
|
||||
self.assertIn("validating the container specification", result["error"])
|
||||
self.assertIn("HTTP 404", result["error"])
|
||||
self.assertIn("at artifact_open:", result["error"])
|
||||
self.assertIn("artifact is still available", result["error"])
|
||||
self.assertFalse(any(call[:2] == ("container", "create") for call in self.docker.calls))
|
||||
self.assertIsNone(self.agent.operation_phase)
|
||||
self.agent.journal.close()
|
||||
self.agent = Agent(self.config, self.docker)
|
||||
self.assertEqual(self.agent.execute(operation), result)
|
||||
with patch.object(self.agent, "request", return_value={}) as report:
|
||||
self.agent.report(result)
|
||||
receipt = report.call_args.args[2]
|
||||
self.assertEqual(receipt["result"]["unexpectedError"], details)
|
||||
safe_output = json.dumps(receipt)
|
||||
for hidden in (secret, "controller.example", "Authorization", self.temp.name, "contentSha256", "rconPassword", "Traceback"):
|
||||
self.assertNotIn(hidden, safe_output)
|
||||
|
||||
def test_unexpected_create_disk_failure_identifies_phase_before_mutation(self):
|
||||
payload = {"game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960}
|
||||
with patch.object(self.agent, "seed_plutonium", side_effect=OSError(module.errno.ENOSPC, "private-error", "/private/secret-path")):
|
||||
result = self.agent.execute(self.operation("instance.create", payload))
|
||||
self.assertEqual(result["state"], "failed", result)
|
||||
self.assertIsNone(result["checkpoint"])
|
||||
details = json.loads(result["result"])["unexpectedError"]
|
||||
self.assertEqual(details["phase"], "seeding-runtime")
|
||||
self.assertEqual(details["errno"], module.errno.ENOSPC)
|
||||
self.assertEqual(details["errnoName"], "ENOSPC")
|
||||
self.assertIn("seeding the runtime cache", result["error"])
|
||||
self.assertIn("free disk space and inodes", result["error"])
|
||||
self.assertNotIn("private", result["error"] + result["result"])
|
||||
self.assertFalse(any(call[:2] == ("container", "create") for call in self.docker.calls))
|
||||
|
||||
def test_unexpected_create_failure_after_checkpoint_remains_unknown_without_replay(self):
|
||||
payload = {"game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960}
|
||||
docker_call = self.docker.call
|
||||
creates = []
|
||||
|
||||
def failed_create(*args, **kwargs):
|
||||
if args[:2] == ("container", "create"):
|
||||
creates.append(args)
|
||||
raise RuntimeError("private-runtime-data") from ValueError("private-cause")
|
||||
return docker_call(*args, **kwargs)
|
||||
|
||||
operation = self.operation("instance.create", payload)
|
||||
with patch.object(self.docker, "call", side_effect=failed_create):
|
||||
result = self.agent.execute(operation)
|
||||
self.assertEqual(result["state"], "unknown", result)
|
||||
self.assertEqual(json.loads(result["checkpoint"])["phase"], "creating")
|
||||
details = json.loads(result["result"])["unexpectedError"]
|
||||
self.assertEqual(details["type"], "RuntimeError")
|
||||
self.assertEqual(details["phase"], "creating-container")
|
||||
self.assertEqual(self.agent.execute(operation), result)
|
||||
self.assertEqual(len(creates), 1)
|
||||
self.assertNotIn("private", result["error"] + result["result"])
|
||||
|
||||
def test_unexpected_error_guidance_uses_safe_exception_metadata(self):
|
||||
for error, category, hint in (
|
||||
(PermissionError(module.errno.EACCES, "private", "/private/path"), "PermissionError", "permissions"),
|
||||
(OSError(module.errno.EDQUOT, "private"), "OSError", "quota"),
|
||||
(OSError(module.errno.EROFS, "private"), "OSError", "read-only"),
|
||||
(module.urllib.error.URLError(ConnectionRefusedError(module.errno.ECONNREFUSED, "private")), "URLError", "connectivity"),
|
||||
(TimeoutError("private"), "TimeoutError", "connectivity"),
|
||||
(http.client.IncompleteRead(b"private", 100), "IncompleteRead", "connectivity"),
|
||||
(module.urllib.error.HTTPError("https://private", 403, "private", {}, None), "HTTPError", "enrollment"),
|
||||
(module.urllib.error.HTTPError("https://private", 503, "private", {}, None), "HTTPError", "controller availability"),
|
||||
):
|
||||
with self.subTest(category=category, hint=hint):
|
||||
message, details = module.unexpected_operation_error(error, "executing")
|
||||
self.assertEqual(details["type"], category)
|
||||
self.assertIn(hint, message)
|
||||
self.assertNotIn("private", message + json.dumps(details))
|
||||
|
||||
def test_unexpected_error_diagnostics_handle_unfamiliar_attributes_without_rendering_them(self):
|
||||
class UnsafeValue:
|
||||
def __str__(self):
|
||||
raise AssertionError("private attribute was rendered")
|
||||
|
||||
def __eq__(self, other):
|
||||
raise AssertionError("private attribute was compared")
|
||||
|
||||
class UnavailableTraceback(Exception):
|
||||
def __getattribute__(self, name):
|
||||
if name == "__traceback__":
|
||||
raise RuntimeError("private traceback access")
|
||||
return super().__getattribute__(name)
|
||||
|
||||
filesystem_error = OSError("private error")
|
||||
filesystem_error.errno = UnsafeValue()
|
||||
errors = [filesystem_error, UnavailableTraceback("private exception"), type("private class", (Exception,), {})("private args"),
|
||||
module.urllib.error.URLError(UnsafeValue()),
|
||||
module.urllib.error.HTTPError("https://private", UnsafeValue(), "private reason", {}, None)]
|
||||
for error in errors:
|
||||
error.__cause__ = RuntimeError("private cause")
|
||||
message, details = module.unexpected_operation_error(error, "private phase")
|
||||
self.assertEqual(details["phase"], "executing")
|
||||
self.assertNotIn("httpStatus", details)
|
||||
self.assertNotIn("errno", details)
|
||||
self.assertNotIn("private", message + json.dumps(details))
|
||||
|
||||
def test_read_and_health_lanes_retry_unexpected_failures_without_logging_secrets(self):
|
||||
for lane in ("read", "health"):
|
||||
with self.subTest(lane=lane):
|
||||
|
||||
Reference in New Issue
Block a user