diff --git a/agent/ec_agent.py b/agent/ec_agent.py index 0110cc7..6c7583c 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -10,6 +10,7 @@ import base64 from concurrent.futures import ThreadPoolExecutor, as_completed import contextlib from datetime import datetime, timezone +import errno import math import hashlib import http.client @@ -28,12 +29,13 @@ import subprocess import sys import threading import time +import types import urllib.error import urllib.parse import urllib.request import uuid -VERSION = "1.8.2" +VERSION = "1.8.3" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -79,6 +81,106 @@ class AutoscalingInvalidated(Rejected): """This removal needs a new policy decision, never a later local retry.""" +OPERATION_ERROR_PHASES = { + "executing": "executing the operation", + "checking-instance": "checking the existing instance", + "validating-create-spec": "validating the container specification", + "preparing-create-directory": "preparing the instance directory", + "seeding-runtime": "seeding the runtime cache", + "writing-create-files": "writing initial files", + "persisting-create-metadata": "saving instance metadata", + "creating-container": "creating the container", + "starting-container": "starting the container", + "recording-created-instance": "recording the started container", + "inspecting-created-instance": "checking the created container", + "removing-old-instance": "removing the old instance directory", +} + + +def unexpected_operation_error(error, phase): + """Return bounded diagnostics without exception text, paths, or runtime data.""" + def attribute(value, name): + try: + return getattr(value, name, None) + except Exception: + return None + + def category(value): + # Names come from this fixed list, never an arbitrary exception class. + for kind in (urllib.error.HTTPError, urllib.error.URLError, http.client.IncompleteRead, + http.client.RemoteDisconnected, http.client.HTTPException, + subprocess.TimeoutExpired, TimeoutError, + PermissionError, FileNotFoundError, FileExistsError, IsADirectoryError, + NotADirectoryError, ConnectionError, OSError, sqlite3.OperationalError, sqlite3.Error, + json.JSONDecodeError, UnicodeError, KeyError, TypeError, ValueError, + MemoryError, RuntimeError): + if isinstance(value, kind): + return kind.__name__ + return "Exception" + + phase = phase if isinstance(phase, str) and phase in OPERATION_ERROR_PHASES else "executing" + details = {"type": category(error), "phase": phase, "agentVersion": VERSION} + evidence = error + if isinstance(error, urllib.error.HTTPError): + status = attribute(error, "code") + if type(status) is int and 100 <= status <= 599: + details["httpStatus"] = status + elif isinstance(error, urllib.error.URLError): + reason = attribute(error, "reason") + if isinstance(reason, BaseException): + details["causeType"] = category(reason) + evidence = reason + number = attribute(evidence, "errno") if isinstance(evidence, OSError) else None + number = number if type(number) is int and number in errno.errorcode else None + if number is not None: + details.update(errno=number, errnoName=errno.errorcode[number]) + + frames = [] + trace = attribute(error, "__traceback__") + while isinstance(trace, types.TracebackType): + # Only agent-owned code locations are safe to report. Exclude source + # lines, filenames, other modules, frame locals, and exception chains. + if trace.tb_frame.f_globals is globals(): + frames.append({"function": trace.tb_frame.f_code.co_name, "line": trace.tb_lineno}) + trace = trace.tb_next + details["frames"] = frames[-8:] + + if details.get("httpStatus") in (401, 403): + guidance = "Check host enrollment and controller artifact access permissions." + elif details.get("httpStatus") == 404: + guidance = "Check that the requested artifact is still available on the controller." + elif isinstance(error, urllib.error.HTTPError): + guidance = "Check controller availability and artifact delivery." + elif number == errno.ENOSPC: + guidance = "Check free disk space and inodes on the host." + elif number == errno.EDQUOT: + guidance = "Check the host filesystem quota." + elif number in (errno.EACCES, errno.EPERM) or isinstance(evidence, PermissionError): + guidance = "Check permissions on the host's managed files and directories." + elif number == errno.EROFS: + guidance = "Check whether the host filesystem is mounted read-only." + elif isinstance(error, (urllib.error.URLError, http.client.HTTPException)) or isinstance(evidence, (TimeoutError, ConnectionError)): + guidance = "Check host network, DNS, TLS, and controller connectivity." + elif isinstance(error, subprocess.TimeoutExpired): + guidance = "Check whether the host service command is stalled or unavailable." + elif isinstance(evidence, FileNotFoundError): + guidance = "Check that the required host files and directories are present." + elif isinstance(evidence, OSError): + guidance = "Check host storage, filesystem access, and service availability." + else: + guidance = "Inspect the saved operation diagnostics and host setup before retrying." + labels = [details["type"]] + if "httpStatus" in details: + labels.append(f"HTTP {details['httpStatus']}") + if "errnoName" in details: + labels.append(f"{details['errnoName']} ({details['errno']})") + if "causeType" in details: + labels.append("cause " + details["causeType"]) + if frames: + labels.append(f"at {frames[-1]['function']}:{frames[-1]['line']}") + return f"Operation error while {OPERATION_ERROR_PHASES[phase]} ({'; '.join(labels)}). {guidance}", details + + def systemctl(*args): try: result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False) @@ -962,6 +1064,7 @@ class Agent: self.plutonium_lock = threading.RLock() self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json") self.executing_id = None + self.operation_phase = None self.lane_lock = threading.Lock() self.lanes = {name: {"thread": None, "lastPollAt": None, "polled": time.monotonic(), "busy": False} for name in ("health-1", "health-2", "read", "write")} @@ -1767,6 +1870,7 @@ class Agent: return True def create(self, instance_id, generation, payload): + self.operation_phase = "checking-instance" self.check_creation_cancellation() directory = self.instance_dir(instance_id, generation) current_marker = directory.parent / "current.json" @@ -1779,12 +1883,17 @@ class Agent: if self.spec(instance_id, generation) != payload: raise AgentError("Instance already exists with a different specification") return existing + self.operation_phase = "validating-create-spec" game, library, image, port, bind, env = self.validate_spec(payload) app = directory / "app" + self.operation_phase = "preparing-create-directory" app.mkdir(parents=True, exist_ok=True) self.make_writable(app, directory=True) + self.operation_phase = "seeding-runtime" self.seed_plutonium(app, image) + self.operation_phase = "writing-create-files" self.write_initial_files(app, payload.get("files", [])) + self.operation_phase = "persisting-create-metadata" atomic_json(directory / "spec.json", payload) env_path = directory / "environment" atomic_bytes(env_path, "".join(f"{key}={value}\n" for key, value in env.items()).encode()) @@ -1800,12 +1909,16 @@ class Agent: args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"]) args.append(image) self.check_creation_cancellation() + self.operation_phase = "creating-container" self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha}) self.docker.call(*args, timeout=120) self.check_creation_cancellation() + self.operation_phase = "starting-container" self.docker.call("container", "start", name) self.check_creation_cancellation() + self.operation_phase = "recording-created-instance" atomic_json(current_marker, {"generation": generation}) + self.operation_phase = "inspecting-created-instance" return self.inspect(instance_id, generation) def check_creation_cancellation(self): @@ -1913,6 +2026,7 @@ class Agent: self.docker.call("container", "rm", name) if kind == "instance.rebuild": result = self.create(instance_id, next_generation, spec) + self.operation_phase = "removing-old-instance" old = self.instance_dir(instance_id, generation) if old.exists(): shutil.rmtree(old, ignore_errors=False) @@ -2507,6 +2621,7 @@ class Agent: except AgentError: return self.journal.update(operation["id"], "accepted", {"blocked": "controller-unavailable", "message": "Waiting for controller contact before executing the operation"}) self.executing_id = operation["id"] + self.operation_phase = "executing" try: result = self.dispatch(operation) except Held as err: @@ -2518,11 +2633,17 @@ class Agent: except Exception as err: # The command may already have changed Docker or files. Never infer # failure means nothing happened, and never retry without inspection. - message = str(err) if isinstance(err, AgentError) else "Operation could not be completed; inspect its current state" + diagnostic = None + if isinstance(err, AgentError): + message = str(err) + else: + message, details = unexpected_operation_error(err, self.operation_phase) + diagnostic = {"unexpectedError": details} checkpoint = self.journal.get(operation["id"]).get("checkpoint") - return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message) + return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", diagnostic, error=message) finally: self.executing_id = None + self.operation_phase = None return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result) def recover_credential(self): diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 6825704..137f224 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -404,6 +404,128 @@ class AgentTests(unittest.TestCase): with self.assertRaisesRegex(AgentError, "^Controller request failed$"): self.agent.request("GET", "/api/agent/operations") + def test_unexpected_create_http_failure_reports_safe_diagnostics_and_does_not_replay(self): + secret = "private-token-must-not-appear" + error = module.urllib.error.HTTPError(f"https://user:{secret}@controller.example/blobs?token={secret}", 404, + secret, {"Authorization": "Bearer " + secret}, None) + error.__cause__ = RuntimeError(secret) + payload = {"game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", + "port": 28960, "rconPassword": secret, "files": [{"path": "configs/server.cfg", "contentSha256": "a" * 64}]} + operation = self.operation("instance.create", payload) + with patch.object(module.urllib.request, "build_opener") as opener: + opener.return_value.open.side_effect = error + result = self.agent.execute(operation) + self.assertEqual(result["state"], "failed", result) + self.assertIsNone(result["checkpoint"]) + self.assertEqual(self.agent.execute(operation), result) + opener.return_value.open.assert_called_once() + details = json.loads(result["result"])["unexpectedError"] + self.assertEqual(details["type"], "HTTPError") + self.assertEqual(details["httpStatus"], 404) + self.assertEqual(details["phase"], "validating-create-spec") + self.assertEqual(details["agentVersion"], module.VERSION) + self.assertEqual(details["frames"][-1]["function"], "artifact_open") + self.assertLessEqual(len(details["frames"]), 8) + self.assertTrue(all(set(frame) == {"function", "line"} and frame["line"] > 0 for frame in details["frames"])) + self.assertIn("validating the container specification", result["error"]) + self.assertIn("HTTP 404", result["error"]) + self.assertIn("at artifact_open:", result["error"]) + self.assertIn("artifact is still available", result["error"]) + self.assertFalse(any(call[:2] == ("container", "create") for call in self.docker.calls)) + self.assertIsNone(self.agent.operation_phase) + self.agent.journal.close() + self.agent = Agent(self.config, self.docker) + self.assertEqual(self.agent.execute(operation), result) + with patch.object(self.agent, "request", return_value={}) as report: + self.agent.report(result) + receipt = report.call_args.args[2] + self.assertEqual(receipt["result"]["unexpectedError"], details) + safe_output = json.dumps(receipt) + for hidden in (secret, "controller.example", "Authorization", self.temp.name, "contentSha256", "rconPassword", "Traceback"): + self.assertNotIn(hidden, safe_output) + + def test_unexpected_create_disk_failure_identifies_phase_before_mutation(self): + payload = {"game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960} + with patch.object(self.agent, "seed_plutonium", side_effect=OSError(module.errno.ENOSPC, "private-error", "/private/secret-path")): + result = self.agent.execute(self.operation("instance.create", payload)) + self.assertEqual(result["state"], "failed", result) + self.assertIsNone(result["checkpoint"]) + details = json.loads(result["result"])["unexpectedError"] + self.assertEqual(details["phase"], "seeding-runtime") + self.assertEqual(details["errno"], module.errno.ENOSPC) + self.assertEqual(details["errnoName"], "ENOSPC") + self.assertIn("seeding the runtime cache", result["error"]) + self.assertIn("free disk space and inodes", result["error"]) + self.assertNotIn("private", result["error"] + result["result"]) + self.assertFalse(any(call[:2] == ("container", "create") for call in self.docker.calls)) + + def test_unexpected_create_failure_after_checkpoint_remains_unknown_without_replay(self): + payload = {"game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960} + docker_call = self.docker.call + creates = [] + + def failed_create(*args, **kwargs): + if args[:2] == ("container", "create"): + creates.append(args) + raise RuntimeError("private-runtime-data") from ValueError("private-cause") + return docker_call(*args, **kwargs) + + operation = self.operation("instance.create", payload) + with patch.object(self.docker, "call", side_effect=failed_create): + result = self.agent.execute(operation) + self.assertEqual(result["state"], "unknown", result) + self.assertEqual(json.loads(result["checkpoint"])["phase"], "creating") + details = json.loads(result["result"])["unexpectedError"] + self.assertEqual(details["type"], "RuntimeError") + self.assertEqual(details["phase"], "creating-container") + self.assertEqual(self.agent.execute(operation), result) + self.assertEqual(len(creates), 1) + self.assertNotIn("private", result["error"] + result["result"]) + + def test_unexpected_error_guidance_uses_safe_exception_metadata(self): + for error, category, hint in ( + (PermissionError(module.errno.EACCES, "private", "/private/path"), "PermissionError", "permissions"), + (OSError(module.errno.EDQUOT, "private"), "OSError", "quota"), + (OSError(module.errno.EROFS, "private"), "OSError", "read-only"), + (module.urllib.error.URLError(ConnectionRefusedError(module.errno.ECONNREFUSED, "private")), "URLError", "connectivity"), + (TimeoutError("private"), "TimeoutError", "connectivity"), + (http.client.IncompleteRead(b"private", 100), "IncompleteRead", "connectivity"), + (module.urllib.error.HTTPError("https://private", 403, "private", {}, None), "HTTPError", "enrollment"), + (module.urllib.error.HTTPError("https://private", 503, "private", {}, None), "HTTPError", "controller availability"), + ): + with self.subTest(category=category, hint=hint): + message, details = module.unexpected_operation_error(error, "executing") + self.assertEqual(details["type"], category) + self.assertIn(hint, message) + self.assertNotIn("private", message + json.dumps(details)) + + def test_unexpected_error_diagnostics_handle_unfamiliar_attributes_without_rendering_them(self): + class UnsafeValue: + def __str__(self): + raise AssertionError("private attribute was rendered") + + def __eq__(self, other): + raise AssertionError("private attribute was compared") + + class UnavailableTraceback(Exception): + def __getattribute__(self, name): + if name == "__traceback__": + raise RuntimeError("private traceback access") + return super().__getattribute__(name) + + filesystem_error = OSError("private error") + filesystem_error.errno = UnsafeValue() + errors = [filesystem_error, UnavailableTraceback("private exception"), type("private class", (Exception,), {})("private args"), + module.urllib.error.URLError(UnsafeValue()), + module.urllib.error.HTTPError("https://private", UnsafeValue(), "private reason", {}, None)] + for error in errors: + error.__cause__ = RuntimeError("private cause") + message, details = module.unexpected_operation_error(error, "private phase") + self.assertEqual(details["phase"], "executing") + self.assertNotIn("httpStatus", details) + self.assertNotIn("errno", details) + self.assertNotIn("private", message + json.dumps(details)) + def test_read_and_health_lanes_retry_unexpected_failures_without_logging_secrets(self): for lane in ("read", "health"): with self.subTest(lane=lane): diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py index 0110cc7..6c7583c 100644 --- a/static/agent/ec-agent.py +++ b/static/agent/ec-agent.py @@ -10,6 +10,7 @@ import base64 from concurrent.futures import ThreadPoolExecutor, as_completed import contextlib from datetime import datetime, timezone +import errno import math import hashlib import http.client @@ -28,12 +29,13 @@ import subprocess import sys import threading import time +import types import urllib.error import urllib.parse import urllib.request import uuid -VERSION = "1.8.2" +VERSION = "1.8.3" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -79,6 +81,106 @@ class AutoscalingInvalidated(Rejected): """This removal needs a new policy decision, never a later local retry.""" +OPERATION_ERROR_PHASES = { + "executing": "executing the operation", + "checking-instance": "checking the existing instance", + "validating-create-spec": "validating the container specification", + "preparing-create-directory": "preparing the instance directory", + "seeding-runtime": "seeding the runtime cache", + "writing-create-files": "writing initial files", + "persisting-create-metadata": "saving instance metadata", + "creating-container": "creating the container", + "starting-container": "starting the container", + "recording-created-instance": "recording the started container", + "inspecting-created-instance": "checking the created container", + "removing-old-instance": "removing the old instance directory", +} + + +def unexpected_operation_error(error, phase): + """Return bounded diagnostics without exception text, paths, or runtime data.""" + def attribute(value, name): + try: + return getattr(value, name, None) + except Exception: + return None + + def category(value): + # Names come from this fixed list, never an arbitrary exception class. + for kind in (urllib.error.HTTPError, urllib.error.URLError, http.client.IncompleteRead, + http.client.RemoteDisconnected, http.client.HTTPException, + subprocess.TimeoutExpired, TimeoutError, + PermissionError, FileNotFoundError, FileExistsError, IsADirectoryError, + NotADirectoryError, ConnectionError, OSError, sqlite3.OperationalError, sqlite3.Error, + json.JSONDecodeError, UnicodeError, KeyError, TypeError, ValueError, + MemoryError, RuntimeError): + if isinstance(value, kind): + return kind.__name__ + return "Exception" + + phase = phase if isinstance(phase, str) and phase in OPERATION_ERROR_PHASES else "executing" + details = {"type": category(error), "phase": phase, "agentVersion": VERSION} + evidence = error + if isinstance(error, urllib.error.HTTPError): + status = attribute(error, "code") + if type(status) is int and 100 <= status <= 599: + details["httpStatus"] = status + elif isinstance(error, urllib.error.URLError): + reason = attribute(error, "reason") + if isinstance(reason, BaseException): + details["causeType"] = category(reason) + evidence = reason + number = attribute(evidence, "errno") if isinstance(evidence, OSError) else None + number = number if type(number) is int and number in errno.errorcode else None + if number is not None: + details.update(errno=number, errnoName=errno.errorcode[number]) + + frames = [] + trace = attribute(error, "__traceback__") + while isinstance(trace, types.TracebackType): + # Only agent-owned code locations are safe to report. Exclude source + # lines, filenames, other modules, frame locals, and exception chains. + if trace.tb_frame.f_globals is globals(): + frames.append({"function": trace.tb_frame.f_code.co_name, "line": trace.tb_lineno}) + trace = trace.tb_next + details["frames"] = frames[-8:] + + if details.get("httpStatus") in (401, 403): + guidance = "Check host enrollment and controller artifact access permissions." + elif details.get("httpStatus") == 404: + guidance = "Check that the requested artifact is still available on the controller." + elif isinstance(error, urllib.error.HTTPError): + guidance = "Check controller availability and artifact delivery." + elif number == errno.ENOSPC: + guidance = "Check free disk space and inodes on the host." + elif number == errno.EDQUOT: + guidance = "Check the host filesystem quota." + elif number in (errno.EACCES, errno.EPERM) or isinstance(evidence, PermissionError): + guidance = "Check permissions on the host's managed files and directories." + elif number == errno.EROFS: + guidance = "Check whether the host filesystem is mounted read-only." + elif isinstance(error, (urllib.error.URLError, http.client.HTTPException)) or isinstance(evidence, (TimeoutError, ConnectionError)): + guidance = "Check host network, DNS, TLS, and controller connectivity." + elif isinstance(error, subprocess.TimeoutExpired): + guidance = "Check whether the host service command is stalled or unavailable." + elif isinstance(evidence, FileNotFoundError): + guidance = "Check that the required host files and directories are present." + elif isinstance(evidence, OSError): + guidance = "Check host storage, filesystem access, and service availability." + else: + guidance = "Inspect the saved operation diagnostics and host setup before retrying." + labels = [details["type"]] + if "httpStatus" in details: + labels.append(f"HTTP {details['httpStatus']}") + if "errnoName" in details: + labels.append(f"{details['errnoName']} ({details['errno']})") + if "causeType" in details: + labels.append("cause " + details["causeType"]) + if frames: + labels.append(f"at {frames[-1]['function']}:{frames[-1]['line']}") + return f"Operation error while {OPERATION_ERROR_PHASES[phase]} ({'; '.join(labels)}). {guidance}", details + + def systemctl(*args): try: result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False) @@ -962,6 +1064,7 @@ class Agent: self.plutonium_lock = threading.RLock() self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json") self.executing_id = None + self.operation_phase = None self.lane_lock = threading.Lock() self.lanes = {name: {"thread": None, "lastPollAt": None, "polled": time.monotonic(), "busy": False} for name in ("health-1", "health-2", "read", "write")} @@ -1767,6 +1870,7 @@ class Agent: return True def create(self, instance_id, generation, payload): + self.operation_phase = "checking-instance" self.check_creation_cancellation() directory = self.instance_dir(instance_id, generation) current_marker = directory.parent / "current.json" @@ -1779,12 +1883,17 @@ class Agent: if self.spec(instance_id, generation) != payload: raise AgentError("Instance already exists with a different specification") return existing + self.operation_phase = "validating-create-spec" game, library, image, port, bind, env = self.validate_spec(payload) app = directory / "app" + self.operation_phase = "preparing-create-directory" app.mkdir(parents=True, exist_ok=True) self.make_writable(app, directory=True) + self.operation_phase = "seeding-runtime" self.seed_plutonium(app, image) + self.operation_phase = "writing-create-files" self.write_initial_files(app, payload.get("files", [])) + self.operation_phase = "persisting-create-metadata" atomic_json(directory / "spec.json", payload) env_path = directory / "environment" atomic_bytes(env_path, "".join(f"{key}={value}\n" for key, value in env.items()).encode()) @@ -1800,12 +1909,16 @@ class Agent: args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"]) args.append(image) self.check_creation_cancellation() + self.operation_phase = "creating-container" self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha}) self.docker.call(*args, timeout=120) self.check_creation_cancellation() + self.operation_phase = "starting-container" self.docker.call("container", "start", name) self.check_creation_cancellation() + self.operation_phase = "recording-created-instance" atomic_json(current_marker, {"generation": generation}) + self.operation_phase = "inspecting-created-instance" return self.inspect(instance_id, generation) def check_creation_cancellation(self): @@ -1913,6 +2026,7 @@ class Agent: self.docker.call("container", "rm", name) if kind == "instance.rebuild": result = self.create(instance_id, next_generation, spec) + self.operation_phase = "removing-old-instance" old = self.instance_dir(instance_id, generation) if old.exists(): shutil.rmtree(old, ignore_errors=False) @@ -2507,6 +2621,7 @@ class Agent: except AgentError: return self.journal.update(operation["id"], "accepted", {"blocked": "controller-unavailable", "message": "Waiting for controller contact before executing the operation"}) self.executing_id = operation["id"] + self.operation_phase = "executing" try: result = self.dispatch(operation) except Held as err: @@ -2518,11 +2633,17 @@ class Agent: except Exception as err: # The command may already have changed Docker or files. Never infer # failure means nothing happened, and never retry without inspection. - message = str(err) if isinstance(err, AgentError) else "Operation could not be completed; inspect its current state" + diagnostic = None + if isinstance(err, AgentError): + message = str(err) + else: + message, details = unexpected_operation_error(err, self.operation_phase) + diagnostic = {"unexpectedError": details} checkpoint = self.journal.get(operation["id"]).get("checkpoint") - return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message) + return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", diagnostic, error=message) finally: self.executing_id = None + self.operation_phase = None return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result) def recover_credential(self): diff --git a/static/agent/manifest.json b/static/agent/manifest.json index 8d3055b..acab6d2 100644 --- a/static/agent/manifest.json +++ b/static/agent/manifest.json @@ -1,8 +1,8 @@ { - "version": "1.8.2", + "version": "1.8.3", "protocolVersion": 1, "platform": "debian-amd64", - "sha256": "9715027ee71b4bab7df3ec44186eb81554f913741f1dde3e5d79b7e525a32ba6", + "sha256": "388569ffd4566dee63584970838d4d3ae7d2b8ef92675ae7627c31389dcbfec0", "updaterProtocolVersion": 1, "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8" }