josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Report safe diagnostics for unexpected agent failures

Browse Source
This commit is contained in:
josh
2026-10-08 14:00:13 -04:00
parent b3fa486886
commit 511f090552
4 changed files with 372 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/ec_agent.py
+124 -3
View File
@@ -10,6 +10,7 @@ import base64
from concurrent.futures import ThreadPoolExecutor, as_completed
import contextlib
from datetime import datetime, timezone
import errno
import math
import hashlib
import http.client
@@ -28,12 +29,13 @@ import subprocess
import sys
import threading
import time
import types
import urllib.error
import urllib.parse
import urllib.request
import uuid
VERSION = "1.8.2"
VERSION = "1.8.3"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
MANIFEST_MAX_ENTRIES = 10000
@@ -79,6 +81,106 @@ class AutoscalingInvalidated(Rejected):
"""This removal needs a new policy decision, never a later local retry."""
OPERATION_ERROR_PHASES = {
"executing": "executing the operation",
"checking-instance": "checking the existing instance",
"validating-create-spec": "validating the container specification",
"preparing-create-directory": "preparing the instance directory",
"seeding-runtime": "seeding the runtime cache",
"writing-create-files": "writing initial files",
"persisting-create-metadata": "saving instance metadata",
"creating-container": "creating the container",
"starting-container": "starting the container",
"recording-created-instance": "recording the started container",
"inspecting-created-instance": "checking the created container",
"removing-old-instance": "removing the old instance directory",
}
def unexpected_operation_error(error, phase):
"""Return bounded diagnostics without exception text, paths, or runtime data."""
def attribute(value, name):
try:
return getattr(value, name, None)
except Exception:
return None
def category(value):
# Names come from this fixed list, never an arbitrary exception class.
for kind in (urllib.error.HTTPError, urllib.error.URLError, http.client.IncompleteRead,
http.client.RemoteDisconnected, http.client.HTTPException,
subprocess.TimeoutExpired, TimeoutError,
PermissionError, FileNotFoundError, FileExistsError, IsADirectoryError,
NotADirectoryError, ConnectionError, OSError, sqlite3.OperationalError, sqlite3.Error,
json.JSONDecodeError, UnicodeError, KeyError, TypeError, ValueError,
MemoryError, RuntimeError):
if isinstance(value, kind):
return kind.__name__
return "Exception"
phase = phase if isinstance(phase, str) and phase in OPERATION_ERROR_PHASES else "executing"
details = {"type": category(error), "phase": phase, "agentVersion": VERSION}
evidence = error
if isinstance(error, urllib.error.HTTPError):
status = attribute(error, "code")
if type(status) is int and 100 <= status <= 599:
details["httpStatus"] = status
elif isinstance(error, urllib.error.URLError):
reason = attribute(error, "reason")
if isinstance(reason, BaseException):
details["causeType"] = category(reason)
evidence = reason
number = attribute(evidence, "errno") if isinstance(evidence, OSError) else None
number = number if type(number) is int and number in errno.errorcode else None
if number is not None:
details.update(errno=number, errnoName=errno.errorcode[number])
frames = []
trace = attribute(error, "__traceback__")
while isinstance(trace, types.TracebackType):
# Only agent-owned code locations are safe to report. Exclude source
# lines, filenames, other modules, frame locals, and exception chains.
if trace.tb_frame.f_globals is globals():
frames.append({"function": trace.tb_frame.f_code.co_name, "line": trace.tb_lineno})
trace = trace.tb_next
details["frames"] = frames[-8:]
if details.get("httpStatus") in (401, 403):
guidance = "Check host enrollment and controller artifact access permissions."
elif details.get("httpStatus") == 404:
guidance = "Check that the requested artifact is still available on the controller."
elif isinstance(error, urllib.error.HTTPError):
guidance = "Check controller availability and artifact delivery."
elif number == errno.ENOSPC:
guidance = "Check free disk space and inodes on the host."
elif number == errno.EDQUOT:
guidance = "Check the host filesystem quota."
elif number in (errno.EACCES, errno.EPERM) or isinstance(evidence, PermissionError):
guidance = "Check permissions on the host's managed files and directories."
elif number == errno.EROFS:
guidance = "Check whether the host filesystem is mounted read-only."
elif isinstance(error, (urllib.error.URLError, http.client.HTTPException)) or isinstance(evidence, (TimeoutError, ConnectionError)):
guidance = "Check host network, DNS, TLS, and controller connectivity."
elif isinstance(error, subprocess.TimeoutExpired):
guidance = "Check whether the host service command is stalled or unavailable."
elif isinstance(evidence, FileNotFoundError):
guidance = "Check that the required host files and directories are present."
elif isinstance(evidence, OSError):
guidance = "Check host storage, filesystem access, and service availability."
else:
guidance = "Inspect the saved operation diagnostics and host setup before retrying."
labels = [details["type"]]
if "httpStatus" in details:
labels.append(f"HTTP {details['httpStatus']}")
if "errnoName" in details:
labels.append(f"{details['errnoName']} ({details['errno']})")
if "causeType" in details:
labels.append("cause " + details["causeType"])
if frames:
labels.append(f"at {frames[-1]['function']}:{frames[-1]['line']}")
return f"Operation error while {OPERATION_ERROR_PHASES[phase]} ({'; '.join(labels)}). {guidance}", details
def systemctl(*args):
try:
result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
@@ -962,6 +1064,7 @@ class Agent:
self.plutonium_lock = threading.RLock()
self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
self.executing_id = None
self.operation_phase = None
self.lane_lock = threading.Lock()
self.lanes = {name: {"thread": None, "lastPollAt": None, "polled": time.monotonic(), "busy": False}
for name in ("health-1", "health-2", "read", "write")}
@@ -1767,6 +1870,7 @@ class Agent:
return True
def create(self, instance_id, generation, payload):
self.operation_phase = "checking-instance"
self.check_creation_cancellation()
directory = self.instance_dir(instance_id, generation)
current_marker = directory.parent / "current.json"
@@ -1779,12 +1883,17 @@ class Agent:
if self.spec(instance_id, generation) != payload:
raise AgentError("Instance already exists with a different specification")
return existing
self.operation_phase = "validating-create-spec"
game, library, image, port, bind, env = self.validate_spec(payload)
app = directory / "app"
self.operation_phase = "preparing-create-directory"
app.mkdir(parents=True, exist_ok=True)
self.make_writable(app, directory=True)
self.operation_phase = "seeding-runtime"
self.seed_plutonium(app, image)
self.operation_phase = "writing-create-files"
self.write_initial_files(app, payload.get("files", []))
self.operation_phase = "persisting-create-metadata"
atomic_json(directory / "spec.json", payload)
env_path = directory / "environment"
atomic_bytes(env_path, "".join(f"{key}={value}\n" for key, value in env.items()).encode())
@@ -1800,12 +1909,16 @@ class Agent:
args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
args.append(image)
self.check_creation_cancellation()
self.operation_phase = "creating-container"
self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha})
self.docker.call(*args, timeout=120)
self.check_creation_cancellation()
self.operation_phase = "starting-container"
self.docker.call("container", "start", name)
self.check_creation_cancellation()
self.operation_phase = "recording-created-instance"
atomic_json(current_marker, {"generation": generation})
self.operation_phase = "inspecting-created-instance"
return self.inspect(instance_id, generation)
def check_creation_cancellation(self):
@@ -1913,6 +2026,7 @@ class Agent:
self.docker.call("container", "rm", name)
if kind == "instance.rebuild":
result = self.create(instance_id, next_generation, spec)
self.operation_phase = "removing-old-instance"
old = self.instance_dir(instance_id, generation)
if old.exists():
shutil.rmtree(old, ignore_errors=False)
@@ -2507,6 +2621,7 @@ class Agent:
except AgentError:
return self.journal.update(operation["id"], "accepted", {"blocked": "controller-unavailable", "message": "Waiting for controller contact before executing the operation"})
self.executing_id = operation["id"]
self.operation_phase = "executing"
try:
result = self.dispatch(operation)
except Held as err:
@@ -2518,11 +2633,17 @@ class Agent:
except Exception as err:
# The command may already have changed Docker or files. Never infer
# failure means nothing happened, and never retry without inspection.
message = str(err) if isinstance(err, AgentError) else "Operation could not be completed; inspect its current state"
diagnostic = None
if isinstance(err, AgentError):
message = str(err)
else:
message, details = unexpected_operation_error(err, self.operation_phase)
diagnostic = {"unexpectedError": details}
checkpoint = self.journal.get(operation["id"]).get("checkpoint")
return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message)
return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", diagnostic, error=message)
finally:
self.executing_id = None
self.operation_phase = None
return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result)
def recover_credential(self):
agent/tests/test_agent.py
+122
View File
@@ -404,6 +404,128 @@ class AgentTests(unittest.TestCase):
with self.assertRaisesRegex(AgentError, "^Controller request failed$"):
self.agent.request("GET", "/api/agent/operations")
def test_unexpected_create_http_failure_reports_safe_diagnostics_and_does_not_replay(self):
secret = "private-token-must-not-appear"
error = module.urllib.error.HTTPError(f"https://user:{secret}@controller.example/blobs?token={secret}", 404,
secret, {"Authorization": "Bearer " + secret}, None)
error.__cause__ = RuntimeError(secret)
payload = {"game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest",
"port": 28960, "rconPassword": secret, "files": [{"path": "configs/server.cfg", "contentSha256": "a" * 64}]}
operation = self.operation("instance.create", payload)
with patch.object(module.urllib.request, "build_opener") as opener:
opener.return_value.open.side_effect = error
result = self.agent.execute(operation)
self.assertEqual(result["state"], "failed", result)
self.assertIsNone(result["checkpoint"])
self.assertEqual(self.agent.execute(operation), result)
opener.return_value.open.assert_called_once()
details = json.loads(result["result"])["unexpectedError"]
self.assertEqual(details["type"], "HTTPError")
self.assertEqual(details["httpStatus"], 404)
self.assertEqual(details["phase"], "validating-create-spec")
self.assertEqual(details["agentVersion"], module.VERSION)
self.assertEqual(details["frames"][-1]["function"], "artifact_open")
self.assertLessEqual(len(details["frames"]), 8)
self.assertTrue(all(set(frame) == {"function", "line"} and frame["line"] > 0 for frame in details["frames"]))
self.assertIn("validating the container specification", result["error"])
self.assertIn("HTTP 404", result["error"])
self.assertIn("at artifact_open:", result["error"])
self.assertIn("artifact is still available", result["error"])
self.assertFalse(any(call[:2] == ("container", "create") for call in self.docker.calls))
self.assertIsNone(self.agent.operation_phase)
self.agent.journal.close()
self.agent = Agent(self.config, self.docker)
self.assertEqual(self.agent.execute(operation), result)
with patch.object(self.agent, "request", return_value={}) as report:
self.agent.report(result)
receipt = report.call_args.args[2]
self.assertEqual(receipt["result"]["unexpectedError"], details)
safe_output = json.dumps(receipt)
for hidden in (secret, "controller.example", "Authorization", self.temp.name, "contentSha256", "rconPassword", "Traceback"):
self.assertNotIn(hidden, safe_output)
def test_unexpected_create_disk_failure_identifies_phase_before_mutation(self):
payload = {"game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960}
with patch.object(self.agent, "seed_plutonium", side_effect=OSError(module.errno.ENOSPC, "private-error", "/private/secret-path")):
result = self.agent.execute(self.operation("instance.create", payload))
self.assertEqual(result["state"], "failed", result)
self.assertIsNone(result["checkpoint"])
details = json.loads(result["result"])["unexpectedError"]
self.assertEqual(details["phase"], "seeding-runtime")
self.assertEqual(details["errno"], module.errno.ENOSPC)
self.assertEqual(details["errnoName"], "ENOSPC")
self.assertIn("seeding the runtime cache", result["error"])
self.assertIn("free disk space and inodes", result["error"])
self.assertNotIn("private", result["error"] + result["result"])
self.assertFalse(any(call[:2] == ("container", "create") for call in self.docker.calls))
def test_unexpected_create_failure_after_checkpoint_remains_unknown_without_replay(self):
payload = {"game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960}
docker_call = self.docker.call
creates = []
def failed_create(*args, **kwargs):
if args[:2] == ("container", "create"):
creates.append(args)
raise RuntimeError("private-runtime-data") from ValueError("private-cause")
return docker_call(*args, **kwargs)
operation = self.operation("instance.create", payload)
with patch.object(self.docker, "call", side_effect=failed_create):
result = self.agent.execute(operation)
self.assertEqual(result["state"], "unknown", result)
self.assertEqual(json.loads(result["checkpoint"])["phase"], "creating")
details = json.loads(result["result"])["unexpectedError"]
self.assertEqual(details["type"], "RuntimeError")
self.assertEqual(details["phase"], "creating-container")
self.assertEqual(self.agent.execute(operation), result)
self.assertEqual(len(creates), 1)
self.assertNotIn("private", result["error"] + result["result"])
def test_unexpected_error_guidance_uses_safe_exception_metadata(self):
for error, category, hint in (
(PermissionError(module.errno.EACCES, "private", "/private/path"), "PermissionError", "permissions"),
(OSError(module.errno.EDQUOT, "private"), "OSError", "quota"),
(OSError(module.errno.EROFS, "private"), "OSError", "read-only"),
(module.urllib.error.URLError(ConnectionRefusedError(module.errno.ECONNREFUSED, "private")), "URLError", "connectivity"),
(TimeoutError("private"), "TimeoutError", "connectivity"),
(http.client.IncompleteRead(b"private", 100), "IncompleteRead", "connectivity"),
(module.urllib.error.HTTPError("https://private", 403, "private", {}, None), "HTTPError", "enrollment"),
(module.urllib.error.HTTPError("https://private", 503, "private", {}, None), "HTTPError", "controller availability"),
):
with self.subTest(category=category, hint=hint):
message, details = module.unexpected_operation_error(error, "executing")
self.assertEqual(details["type"], category)
self.assertIn(hint, message)
self.assertNotIn("private", message + json.dumps(details))
def test_unexpected_error_diagnostics_handle_unfamiliar_attributes_without_rendering_them(self):
class UnsafeValue:
def __str__(self):
raise AssertionError("private attribute was rendered")
def __eq__(self, other):
raise AssertionError("private attribute was compared")
class UnavailableTraceback(Exception):
def __getattribute__(self, name):
if name == "__traceback__":
raise RuntimeError("private traceback access")
return super().__getattribute__(name)
filesystem_error = OSError("private error")
filesystem_error.errno = UnsafeValue()
errors = [filesystem_error, UnavailableTraceback("private exception"), type("private class", (Exception,), {})("private args"),
module.urllib.error.URLError(UnsafeValue()),
module.urllib.error.HTTPError("https://private", UnsafeValue(), "private reason", {}, None)]
for error in errors:
error.__cause__ = RuntimeError("private cause")
message, details = module.unexpected_operation_error(error, "private phase")
self.assertEqual(details["phase"], "executing")
self.assertNotIn("httpStatus", details)
self.assertNotIn("errno", details)
self.assertNotIn("private", message + json.dumps(details))
def test_read_and_health_lanes_retry_unexpected_failures_without_logging_secrets(self):
for lane in ("read", "health"):
with self.subTest(lane=lane):
static/agent/ec-agent.py
+124 -3
View File
@@ -10,6 +10,7 @@ import base64
from concurrent.futures import ThreadPoolExecutor, as_completed
import contextlib
from datetime import datetime, timezone
import errno
import math
import hashlib
import http.client
@@ -28,12 +29,13 @@ import subprocess
import sys
import threading
import time
import types
import urllib.error
import urllib.parse
import urllib.request
import uuid
VERSION = "1.8.2"
VERSION = "1.8.3"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
MANIFEST_MAX_ENTRIES = 10000
@@ -79,6 +81,106 @@ class AutoscalingInvalidated(Rejected):
"""This removal needs a new policy decision, never a later local retry."""
OPERATION_ERROR_PHASES = {
"executing": "executing the operation",
"checking-instance": "checking the existing instance",
"validating-create-spec": "validating the container specification",
"preparing-create-directory": "preparing the instance directory",
"seeding-runtime": "seeding the runtime cache",
"writing-create-files": "writing initial files",
"persisting-create-metadata": "saving instance metadata",
"creating-container": "creating the container",
"starting-container": "starting the container",
"recording-created-instance": "recording the started container",
"inspecting-created-instance": "checking the created container",
"removing-old-instance": "removing the old instance directory",
}
def unexpected_operation_error(error, phase):
"""Return bounded diagnostics without exception text, paths, or runtime data."""
def attribute(value, name):
try:
return getattr(value, name, None)
except Exception:
return None
def category(value):
# Names come from this fixed list, never an arbitrary exception class.
for kind in (urllib.error.HTTPError, urllib.error.URLError, http.client.IncompleteRead,
http.client.RemoteDisconnected, http.client.HTTPException,
subprocess.TimeoutExpired, TimeoutError,
PermissionError, FileNotFoundError, FileExistsError, IsADirectoryError,
NotADirectoryError, ConnectionError, OSError, sqlite3.OperationalError, sqlite3.Error,
json.JSONDecodeError, UnicodeError, KeyError, TypeError, ValueError,
MemoryError, RuntimeError):
if isinstance(value, kind):
return kind.__name__
return "Exception"
phase = phase if isinstance(phase, str) and phase in OPERATION_ERROR_PHASES else "executing"
details = {"type": category(error), "phase": phase, "agentVersion": VERSION}
evidence = error
if isinstance(error, urllib.error.HTTPError):
status = attribute(error, "code")
if type(status) is int and 100 <= status <= 599:
details["httpStatus"] = status
elif isinstance(error, urllib.error.URLError):
reason = attribute(error, "reason")
if isinstance(reason, BaseException):
details["causeType"] = category(reason)
evidence = reason
number = attribute(evidence, "errno") if isinstance(evidence, OSError) else None
number = number if type(number) is int and number in errno.errorcode else None
if number is not None:
details.update(errno=number, errnoName=errno.errorcode[number])
frames = []
trace = attribute(error, "__traceback__")
while isinstance(trace, types.TracebackType):
# Only agent-owned code locations are safe to report. Exclude source
# lines, filenames, other modules, frame locals, and exception chains.
if trace.tb_frame.f_globals is globals():
frames.append({"function": trace.tb_frame.f_code.co_name, "line": trace.tb_lineno})
trace = trace.tb_next
details["frames"] = frames[-8:]
if details.get("httpStatus") in (401, 403):
guidance = "Check host enrollment and controller artifact access permissions."
elif details.get("httpStatus") == 404:
guidance = "Check that the requested artifact is still available on the controller."
elif isinstance(error, urllib.error.HTTPError):
guidance = "Check controller availability and artifact delivery."
elif number == errno.ENOSPC:
guidance = "Check free disk space and inodes on the host."
elif number == errno.EDQUOT:
guidance = "Check the host filesystem quota."
elif number in (errno.EACCES, errno.EPERM) or isinstance(evidence, PermissionError):
guidance = "Check permissions on the host's managed files and directories."
elif number == errno.EROFS:
guidance = "Check whether the host filesystem is mounted read-only."
elif isinstance(error, (urllib.error.URLError, http.client.HTTPException)) or isinstance(evidence, (TimeoutError, ConnectionError)):
guidance = "Check host network, DNS, TLS, and controller connectivity."
elif isinstance(error, subprocess.TimeoutExpired):
guidance = "Check whether the host service command is stalled or unavailable."
elif isinstance(evidence, FileNotFoundError):
guidance = "Check that the required host files and directories are present."
elif isinstance(evidence, OSError):
guidance = "Check host storage, filesystem access, and service availability."
else:
guidance = "Inspect the saved operation diagnostics and host setup before retrying."
labels = [details["type"]]
if "httpStatus" in details:
labels.append(f"HTTP {details['httpStatus']}")
if "errnoName" in details:
labels.append(f"{details['errnoName']} ({details['errno']})")
if "causeType" in details:
labels.append("cause " + details["causeType"])
if frames:
labels.append(f"at {frames[-1]['function']}:{frames[-1]['line']}")
return f"Operation error while {OPERATION_ERROR_PHASES[phase]} ({'; '.join(labels)}). {guidance}", details
def systemctl(*args):
try:
result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
@@ -962,6 +1064,7 @@ class Agent:
self.plutonium_lock = threading.RLock()
self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
self.executing_id = None
self.operation_phase = None
self.lane_lock = threading.Lock()
self.lanes = {name: {"thread": None, "lastPollAt": None, "polled": time.monotonic(), "busy": False}
for name in ("health-1", "health-2", "read", "write")}
@@ -1767,6 +1870,7 @@ class Agent:
return True
def create(self, instance_id, generation, payload):
self.operation_phase = "checking-instance"
self.check_creation_cancellation()
directory = self.instance_dir(instance_id, generation)
current_marker = directory.parent / "current.json"
@@ -1779,12 +1883,17 @@ class Agent:
if self.spec(instance_id, generation) != payload:
raise AgentError("Instance already exists with a different specification")
return existing
self.operation_phase = "validating-create-spec"
game, library, image, port, bind, env = self.validate_spec(payload)
app = directory / "app"
self.operation_phase = "preparing-create-directory"
app.mkdir(parents=True, exist_ok=True)
self.make_writable(app, directory=True)
self.operation_phase = "seeding-runtime"
self.seed_plutonium(app, image)
self.operation_phase = "writing-create-files"
self.write_initial_files(app, payload.get("files", []))
self.operation_phase = "persisting-create-metadata"
atomic_json(directory / "spec.json", payload)
env_path = directory / "environment"
atomic_bytes(env_path, "".join(f"{key}={value}\n" for key, value in env.items()).encode())
@@ -1800,12 +1909,16 @@ class Agent:
args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
args.append(image)
self.check_creation_cancellation()
self.operation_phase = "creating-container"
self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha})
self.docker.call(*args, timeout=120)
self.check_creation_cancellation()
self.operation_phase = "starting-container"
self.docker.call("container", "start", name)
self.check_creation_cancellation()
self.operation_phase = "recording-created-instance"
atomic_json(current_marker, {"generation": generation})
self.operation_phase = "inspecting-created-instance"
return self.inspect(instance_id, generation)
def check_creation_cancellation(self):
@@ -1913,6 +2026,7 @@ class Agent:
self.docker.call("container", "rm", name)
if kind == "instance.rebuild":
result = self.create(instance_id, next_generation, spec)
self.operation_phase = "removing-old-instance"
old = self.instance_dir(instance_id, generation)
if old.exists():
shutil.rmtree(old, ignore_errors=False)
@@ -2507,6 +2621,7 @@ class Agent:
except AgentError:
return self.journal.update(operation["id"], "accepted", {"blocked": "controller-unavailable", "message": "Waiting for controller contact before executing the operation"})
self.executing_id = operation["id"]
self.operation_phase = "executing"
try:
result = self.dispatch(operation)
except Held as err:
@@ -2518,11 +2633,17 @@ class Agent:
except Exception as err:
# The command may already have changed Docker or files. Never infer
# failure means nothing happened, and never retry without inspection.
message = str(err) if isinstance(err, AgentError) else "Operation could not be completed; inspect its current state"
diagnostic = None
if isinstance(err, AgentError):
message = str(err)
else:
message, details = unexpected_operation_error(err, self.operation_phase)
diagnostic = {"unexpectedError": details}
checkpoint = self.journal.get(operation["id"]).get("checkpoint")
return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message)
return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", diagnostic, error=message)
finally:
self.executing_id = None
self.operation_phase = None
return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result)
def recover_credential(self):
static/agent/manifest.json
+2 -2
View File
@@ -1,8 +1,8 @@
{
"version": "1.8.2",
"version": "1.8.3",
"protocolVersion": 1,
"platform": "debian-amd64",
"sha256": "9715027ee71b4bab7df3ec44186eb81554f913741f1dde3e5d79b7e525a32ba6",
"sha256": "388569ffd4566dee63584970838d4d3ae7d2b8ef92675ae7627c31389dcbfec0",
"updaterProtocolVersion": 1,
"updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
}