Add automatic agent updates with verified rollback
This commit is contained in:
+124
-19
@@ -29,7 +29,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.1.0"
|
||||
VERSION = "1.2.0"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
|
||||
@@ -39,6 +39,13 @@ AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service")
|
||||
UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
|
||||
UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service")
|
||||
UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
|
||||
UPDATE_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-update.py")
|
||||
UPDATE_UNIT = Path("/etc/systemd/system/electric-cherry-update.service")
|
||||
UPDATE_TIMER = Path("/etc/systemd/system/electric-cherry-update.timer")
|
||||
UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-cherry-update.timer")
|
||||
UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
|
||||
UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
|
||||
AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
|
||||
ROOT_MARKER = ".electric-cherry.json"
|
||||
|
||||
|
||||
@@ -105,7 +112,7 @@ def uninstall_root(value, identity=None):
|
||||
forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"}
|
||||
if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden:
|
||||
raise AgentError("Host data directory is unsafe to remove")
|
||||
for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT):
|
||||
for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT, UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER, AGENT_EXECUTABLE.with_suffix(".py.update"), UNINSTALL_EXECUTABLE.with_suffix(".py.update")):
|
||||
if installed == root or root in installed.parents:
|
||||
raise AgentError("Host data directory overlaps the agent installation")
|
||||
protected_path(root, missing=True)
|
||||
@@ -499,6 +506,19 @@ class Uninstaller:
|
||||
|
||||
def remove_data(self):
|
||||
protected_path(AGENT_CONFIG.parent, missing=True)
|
||||
# Prevent a scheduled update from recreating agent files during removal.
|
||||
# Stopping the worker is safe here: the installation and any backups are
|
||||
# being removed together, and the durable decommission request blocks
|
||||
# future update attempts.
|
||||
if not self.job.get("updaterStopped"):
|
||||
for unit in (UPDATE_TIMER, UPDATE_UNIT):
|
||||
protected_file(unit, missing=True)
|
||||
if unit.exists():
|
||||
if unit == UPDATE_TIMER:
|
||||
systemctl("disable", "--now", unit.name)
|
||||
else:
|
||||
systemctl("stop", unit.name)
|
||||
self.save(updaterStopped=True)
|
||||
if not self.job.get("agentStopped"):
|
||||
systemctl("disable", "--now", AGENT_UNIT.name)
|
||||
self.save(agentStopped=True)
|
||||
@@ -512,7 +532,7 @@ class Uninstaller:
|
||||
raise AgentError("This platform cannot safely remove managed host files")
|
||||
shutil.rmtree(root)
|
||||
sync_parent(root)
|
||||
for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT):
|
||||
for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER, AGENT_EXECUTABLE.with_suffix(".py.update"), UNINSTALL_EXECUTABLE.with_suffix(".py.update")):
|
||||
protected_file(path, missing=True)
|
||||
path.unlink(missing_ok=True)
|
||||
sync_parent(path)
|
||||
@@ -591,6 +611,74 @@ class Agent:
|
||||
except (AgentError, OSError, ValueError):
|
||||
return False
|
||||
|
||||
def can_update(self):
|
||||
try:
|
||||
if Path(self.config_path) != AGENT_CONFIG:
|
||||
return False
|
||||
uninstall_ownership(self.config)
|
||||
for path in (UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER):
|
||||
protected_file(path)
|
||||
protected_path(UPDATE_TIMER_ENABLED.parent)
|
||||
return UPDATE_TIMER_ENABLED.is_symlink() and UPDATE_TIMER_ENABLED.resolve() == UPDATE_TIMER
|
||||
except (AgentError, OSError, ValueError):
|
||||
return False
|
||||
|
||||
def update_inventory(self):
|
||||
result = {"enabled": self.can_update(), "state": "current", "currentVersion": VERSION}
|
||||
if Path(self.config_path) != AGENT_CONFIG:
|
||||
return result
|
||||
try:
|
||||
protected_file(UPDATE_STATUS, missing=True)
|
||||
if not UPDATE_STATUS.exists():
|
||||
return result
|
||||
if UPDATE_STATUS.stat().st_size > 16384:
|
||||
raise ValueError("Invalid update status")
|
||||
saved = json.loads(UPDATE_STATUS.read_text())
|
||||
if saved.get("state") in ("current", "checking", "deferred", "updating", "updated", "failed"):
|
||||
result["state"] = saved["state"]
|
||||
for key in ("targetVersion", "error"):
|
||||
if isinstance(saved.get(key), str):
|
||||
result[key] = saved[key][:2000 if key == "error" else 80]
|
||||
checked_at = saved.get("checkedAt")
|
||||
if isinstance(checked_at, (int, float)) and not isinstance(checked_at, bool):
|
||||
result["checkedAt"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(checked_at))
|
||||
elif isinstance(checked_at, str):
|
||||
result["checkedAt"] = checked_at[:80]
|
||||
except (AgentError, OSError, ValueError, TypeError, AttributeError, OverflowError):
|
||||
result.update(state="failed", error="Agent update status is unavailable; inspect the update service.")
|
||||
return result
|
||||
|
||||
def update_pending(self):
|
||||
# A persisted transaction survives crashes and is cleared only by the
|
||||
# independent worker after health confirmation or a completed rollback.
|
||||
return Path(self.config_path) == AGENT_CONFIG and (UPDATE_JOURNAL.exists() or UPDATE_JOURNAL.is_symlink())
|
||||
|
||||
@contextlib.contextmanager
|
||||
def activity(self):
|
||||
path = self.root / "agent" / "activity.lock"
|
||||
descriptor = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
|
||||
try:
|
||||
try:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
yield False
|
||||
return
|
||||
yield True
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
def record_health(self):
|
||||
if Path(self.config_path) != AGENT_CONFIG or not self.can_update():
|
||||
return
|
||||
value = {"version": VERSION, "pid": os.getpid(), "checkedAt": time.time()}
|
||||
protected_file(AGENT_HEALTH, missing=True)
|
||||
if self.update_pending():
|
||||
protected_file(UPDATE_JOURNAL)
|
||||
journal = json.loads(UPDATE_JOURNAL.read_text())
|
||||
if isinstance(journal.get("id"), str):
|
||||
value["updateId"] = journal["id"]
|
||||
atomic_json(AGENT_HEALTH, value)
|
||||
|
||||
def decommission(self, operation):
|
||||
payload = operation.get("payload", {})
|
||||
if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]:
|
||||
@@ -733,7 +821,9 @@ class Agent:
|
||||
for line in Path("/etc/os-release").read_text().splitlines():
|
||||
if line.startswith("PRETTY_NAME="):
|
||||
os_name = line.split("=", 1)[1].strip('"')
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
update = self.update_inventory()
|
||||
capabilities = (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
|
||||
def safe_to_disrupt(self, instance_id, generation):
|
||||
health = self.inspect(instance_id, generation)
|
||||
@@ -1462,40 +1552,55 @@ class Agent:
|
||||
host = response.get("host", {})
|
||||
if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int):
|
||||
self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]}
|
||||
# Written only after the controller accepted this process's authenticated
|
||||
# heartbeat. Service liveness alone cannot prove an update works.
|
||||
self.record_health()
|
||||
|
||||
def heartbeat_loop(self):
|
||||
while not self.stopping.wait(15):
|
||||
try:
|
||||
self.heartbeat()
|
||||
except AgentError:
|
||||
except (AgentError, OSError, ValueError):
|
||||
pass
|
||||
|
||||
def poll_operations(self):
|
||||
with self.activity() as acquired:
|
||||
if not acquired or self.stopping.is_set() or self.update_pending():
|
||||
return
|
||||
if self.config.get("pendingCredential"):
|
||||
self.recover_credential()
|
||||
response = self.request("GET", "/api/agent/operations")
|
||||
for operation in response.get("operations", []):
|
||||
if self.stopping.is_set():
|
||||
break
|
||||
self.current_operation = operation["id"]
|
||||
try:
|
||||
accepted, _ = self.journal.accept(operation)
|
||||
self.report(accepted)
|
||||
self.report(self.execute(operation, report_running=True))
|
||||
finally:
|
||||
self.current_operation = None
|
||||
|
||||
def run(self):
|
||||
if self.config.get("pendingCredential"):
|
||||
self.recover_credential()
|
||||
# Obtain the configured host port range before accepting any deployment.
|
||||
# Enrollment's default values must never race the first heartbeat.
|
||||
while not self.stopping.is_set():
|
||||
try:
|
||||
with self.activity() as acquired:
|
||||
if not acquired:
|
||||
self.stopping.wait(3)
|
||||
continue
|
||||
if self.config.get("pendingCredential"):
|
||||
self.recover_credential()
|
||||
self.heartbeat()
|
||||
break
|
||||
except AgentError:
|
||||
except (AgentError, OSError, ValueError):
|
||||
self.stopping.wait(3)
|
||||
heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
|
||||
heartbeats.start()
|
||||
while not self.stopping.is_set():
|
||||
try:
|
||||
if self.config.get("pendingCredential"):
|
||||
self.recover_credential()
|
||||
response = self.request("GET", "/api/agent/operations")
|
||||
for operation in response.get("operations", []):
|
||||
if self.stopping.is_set():
|
||||
break
|
||||
self.current_operation = operation["id"]
|
||||
accepted, _ = self.journal.accept(operation)
|
||||
self.report(accepted)
|
||||
self.report(self.execute(operation, report_running=True))
|
||||
self.current_operation = None
|
||||
self.poll_operations()
|
||||
except (AgentError, OSError, ValueError, KeyError):
|
||||
# Logs deliberately omit URLs, tokens, payloads and raw errors.
|
||||
print("Host controller contact unavailable; existing containers continue running", file=sys.stderr)
|
||||
|
||||
Reference in New Issue
Block a user