Add automatic agent updates with verified rollback

static/agent/ec-agent-update.py

@@ -0,0 +1,479 @@
+#!/usr/bin/env python3
+"""Stable Electric Cherry agent updater; only verified agent code is replaced.
+
+The installer owns this bootstrap and its systemd timer. Updates never execute
+downloaded code before installing it, alter game containers, or edit enrollment
+credentials. A durable journal makes an interrupted switch roll back safely.
+"""
+
+import ast
+import contextlib
+import fcntl
+import hashlib
+import http.client
+import json
+import math
+import os
+from pathlib import Path
+import re
+import stat
+import subprocess
+import sys
+import time
+import urllib.parse
+import urllib.request
+import uuid
+
+UPDATER_PROTOCOL_VERSION = 1
+AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
+AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
+UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
+UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
+UPDATE_LOCK = Path("/etc/electric-cherry/update.lock")
+UPDATE_DIR = Path("/etc/electric-cherry/update")
+UPDATE_JOURNAL = UPDATE_DIR / "journal.json"
+UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
+AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
+AGENT_SERVICE = "electric-cherry-agent.service"
+MAX_MANIFEST = 16 * 1024
+MAX_SOURCE = 8 * 1024 * 1024
+HEALTH_TIMEOUT = 90
+VERSION_RE = re.compile(r"^(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})$")
+DIGEST_RE = re.compile(r"^[0-9a-f]{64}$")
+
+
+class UpdateError(Exception):
+    pass
+
+
+class IntegrityError(UpdateError):
+    pass
+
+
+class ContactError(UpdateError):
+    pass
+
+
+class Deferred(UpdateError):
+    pass
+
+
+def trusted(path, missing=False, directory=False):
+    path = Path(path)
+    for current in (*reversed(path.parents), path):
+        try:
+            info = current.lstat()
+        except FileNotFoundError:
+            if missing:
+                continue
+            raise UpdateError("Agent update files are unavailable; repair the installation") from None
+        expected_type = stat.S_ISDIR if current != path or directory else stat.S_ISREG
+        if not expected_type(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+            raise UpdateError("Agent update paths must be regular root-owned files and directories that other users cannot modify")
+    return path
+
+
+def sync_parent(path):
+    descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def atomic_bytes(path, value, mode=0o600):
+    path = trusted(path, missing=True)
+    trusted(path.parent, directory=True)
+    # Code staging uses known names so recovery and host removal can also
+    # remove a file left behind by a power loss during the atomic write.
+    temporary = path.with_name(path.name + ".update") if path in (AGENT_EXECUTABLE, UNINSTALL_EXECUTABLE) else path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp")
+    try:
+        if temporary.exists() or temporary.is_symlink():
+            trusted(temporary)
+            temporary.unlink()
+            sync_parent(temporary)
+        descriptor = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, mode)
+        with os.fdopen(descriptor, "wb") as stream:
+            stream.write(value)
+            stream.flush()
+            os.fsync(stream.fileno())
+        trusted(path, missing=True)
+        os.replace(temporary, path)
+        sync_parent(path)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def atomic_json(path, value):
+    atomic_bytes(path, json.dumps(value, sort_keys=True, allow_nan=False).encode())
+
+
+def read_bytes(path, limit):
+    trusted(path)
+    with open(path, "rb") as stream:
+        value = stream.read(limit + 1)
+    if len(value) > limit:
+        raise UpdateError("An agent update file exceeds its size limit")
+    return value
+
+
+def read_json(path, limit=MAX_MANIFEST):
+    try:
+        value = json.loads(read_bytes(path, limit))
+    except (ValueError, UnicodeError):
+        raise UpdateError("Agent update metadata is invalid") from None
+    if not isinstance(value, dict):
+        raise UpdateError("Agent update metadata is invalid")
+    return value
+
+
+def sha256(value):
+    return hashlib.sha256(value).hexdigest()
+
+
+def version(value):
+    if not isinstance(value, str) or not VERSION_RE.fullmatch(value):
+        raise IntegrityError("Agent releases must use a numeric major.minor.patch version")
+    return tuple(int(part) for part in value.split("."))
+
+
+def source_version(source):
+    try:
+        tree = ast.parse(source, filename="electric-cherry-agent.py")
+        compile(tree, "electric-cherry-agent.py", "exec")
+    except (SyntaxError, ValueError, TypeError, RecursionError, MemoryError):
+        raise IntegrityError("The agent release is not valid Python source") from None
+    declarations = [node.value.value for node in tree.body if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == "VERSION" for target in node.targets) and isinstance(node.value, ast.Constant)]
+    if len(declarations) != 1:
+        raise IntegrityError("The agent release must declare one explicit VERSION")
+    version(declarations[0])
+    return declarations[0]
+
+
+@contextlib.contextmanager
+def exclusive_lock(path):
+    trusted(path, missing=True)
+    trusted(Path(path).parent, directory=True)
+    descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
+    with os.fdopen(descriptor, "r+") as stream:
+        try:
+            fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            yield None
+            return
+        try:
+            yield stream
+        finally:
+            fcntl.flock(stream, fcntl.LOCK_UN)
+
+
+def systemctl(*args):
+    try:
+        result = subprocess.run(["systemctl", *args], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=60, check=False)
+    except (OSError, subprocess.TimeoutExpired):
+        raise UpdateError("Agent service management is unavailable") from None
+    if result.returncode:
+        raise UpdateError("Agent service management did not complete")
+    return result.stdout.decode("utf-8", "replace")
+
+
+def service_state():
+    output = systemctl("show", AGENT_SERVICE, "--property=ActiveState", "--property=MainPID")
+    fields = dict(line.split("=", 1) for line in output.splitlines() if "=" in line)
+    try:
+        return fields.get("ActiveState") == "active", int(fields.get("MainPID", "0"))
+    except ValueError:
+        return False, 0
+
+
+class Updater:
+    def __init__(self):
+        self.installation = None
+        self.current_version = None
+        self.target = None
+
+    def load_installation(self):
+        config = read_json(AGENT_CONFIG, 64 * 1024)
+        controller = config.get("controller")
+        root_value = config.get("root")
+        host_id = config.get("hostId")
+        if not isinstance(controller, str) or not isinstance(root_value, str) or not isinstance(host_id, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,127}", host_id):
+            raise UpdateError("Agent enrollment is incomplete")
+        parsed = urllib.parse.urlsplit(controller)
+        if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise UpdateError("Agent updates require an HTTPS controller without embedded credentials")
+        root = Path(root_value)
+        if not re.fullmatch(r"/[A-Za-z0-9/_.-]+", root_value) or str(root) != root_value or ".." in root.parts or len(root.parts) < 3:
+            raise UpdateError("The enrolled agent data directory is invalid")
+        trusted(root, directory=True)
+        trusted(root / "agent", directory=True)
+        marker = read_json(root / ".electric-cherry.json")
+        controller = controller.rstrip("/")
+        expected = {"version": 1, "root": root_value, "hostId": host_id, "controller": controller}
+        if marker != expected:
+            raise UpdateError("Agent data ownership does not match this enrollment")
+        info = root.stat()
+        installation = {"hostId": host_id, "controller": controller, "root": root_value, "rootIdentity": [info.st_dev, info.st_ino]}
+        if self.installation is not None and installation != self.installation:
+            raise UpdateError("Agent enrollment changed during the update; no code was replaced")
+        self.installation = installation
+        return installation
+
+    def ensure_directory(self):
+        trusted(UPDATE_DIR, missing=True, directory=True)
+        trusted(UPDATE_DIR.parent, directory=True)
+        if not UPDATE_DIR.exists():
+            UPDATE_DIR.mkdir(mode=0o700)
+            sync_parent(UPDATE_DIR)
+
+    def check_uninstall(self):
+        if UNINSTALL_REQUEST.exists() or UNINSTALL_REQUEST.is_symlink():
+            raise Deferred("Host deletion is pending; agent updates are paused")
+
+    def status(self, state, error=None, current=None):
+        value = {"state": state, "checkedAt": time.time()}
+        if current or self.current_version:
+            value["currentVersion"] = current or self.current_version
+        if self.target:
+            value["targetVersion"] = self.target["version"]
+        if error:
+            value["error"] = str(error)[:500]
+        atomic_json(UPDATE_STATUS, value)
+        return state
+
+    def fetch(self, path, limit):
+        request = urllib.request.Request(self.installation["controller"] + path, headers={"User-Agent": "electric-cherry-updater/1"})
+        class NoRedirect(urllib.request.HTTPRedirectHandler):
+            def redirect_request(self, req, fp, code, msg, headers, newurl):
+                raise ContactError("Agent release redirects are not allowed")
+        try:
+            with urllib.request.build_opener(NoRedirect()).open(request, timeout=25) as response:
+                if response.status != 200:
+                    raise ContactError("The controller did not return a complete agent release")
+                length = response.headers.get("Content-Length")
+                if length is not None and (not length.isdecimal() or int(length) > limit):
+                    raise IntegrityError("The controller release exceeds its download size limit")
+                chunks, total = [], 0
+                deadline = time.monotonic() + 120
+                while True:
+                    if time.monotonic() >= deadline:
+                        raise ContactError("The controller release download timed out")
+                    # HTTPResponse.read1 performs one socket read. read(n) can
+                    # wait indefinitely for n bytes from a trickling server.
+                    block = response.read1(min(64 * 1024, limit + 1 - total))
+                    if not block:
+                        break
+                    total += len(block)
+                    if total > limit:
+                        raise IntegrityError("The controller release exceeds its download size limit")
+                    chunks.append(block)
+                if length is not None and total != int(length):
+                    raise ContactError("The controller release download was interrupted")
+                return b"".join(chunks)
+        except (OSError, ValueError, http.client.HTTPException):
+            raise ContactError("The controller release is unavailable; the running agent was left unchanged") from None
+
+    def manifest(self):
+        try:
+            value = json.loads(self.fetch("/api/agent/release/manifest.json", MAX_MANIFEST))
+        except (ValueError, UnicodeError):
+            raise IntegrityError("The controller release manifest is invalid") from None
+        if not isinstance(value, dict) or type(value.get("protocolVersion")) is not int or value["protocolVersion"] != 1 or value.get("platform") != "debian-amd64":
+            raise IntegrityError("The controller release is not compatible with this host agent")
+        if type(value.get("updaterProtocolVersion")) is not int or value["updaterProtocolVersion"] != UPDATER_PROTOCOL_VERSION:
+            raise Deferred("This release needs a newer updater bootstrap; rerun the host installer")
+        version(value.get("version"))
+        if not isinstance(value.get("sha256"), str) or not DIGEST_RE.fullmatch(value["sha256"]):
+            raise IntegrityError("The controller release manifest has an invalid checksum")
+        return {"version": value["version"], "sha256": value["sha256"]}
+
+    def hold(self, target, error):
+        self.ensure_directory()
+        atomic_json(UPDATE_DIR / "failed-release.json", {**target, "error": error, "failedAt": time.time()})
+
+    def release_held(self):
+        path = UPDATE_DIR / "failed-release.json"
+        if not path.exists():
+            return False
+        held = read_json(path)
+        return all(held.get(key) == self.target[key] for key in ("version", "sha256"))
+
+    def save_journal(self, journal, **changes):
+        journal.update(changes)
+        atomic_json(UPDATE_JOURNAL, journal)
+
+    def read_journal(self):
+        journal = read_json(UPDATE_JOURNAL)
+        if journal.get("installation") != self.installation or journal.get("phase") not in ("installing", "verifying", "rollback") or not isinstance(journal.get("id"), str) or not re.fullmatch(r"[a-f0-9]{32}", journal["id"]):
+            raise UpdateError("Pending agent update ownership or recovery metadata is invalid")
+        version(journal.get("previousVersion"))
+        version(journal.get("targetVersion"))
+        for field in ("targetSha256", "agentBackupSha256", "uninstallBackupSha256"):
+            if not isinstance(journal.get(field), str) or not DIGEST_RE.fullmatch(journal[field]):
+                raise UpdateError("Pending agent update recovery checksums are invalid")
+        return journal
+
+    def finish_journal(self):
+        trusted(UPDATE_JOURNAL)
+        UPDATE_JOURNAL.unlink()
+        sync_parent(UPDATE_JOURNAL)
+        # Removing the journal commits the decision. Backups are disposable
+        # only afterward, so a crash never leaves recovery without its inputs.
+        for name in ("agent.backup.py", "uninstall.backup.py", "candidate.py"):
+            path = trusted(UPDATE_DIR / name, missing=True)
+            path.unlink(missing_ok=True)
+        sync_parent(UPDATE_JOURNAL)
+
+    def activity_lock(self):
+        return exclusive_lock(Path(self.installation["root"]) / "agent" / "activity.lock")
+
+    def install(self, source):
+        with self.activity_lock() as activity:
+            if activity is None:
+                raise Deferred("The agent is executing an operation; update will retry when it is idle")
+            self.check_uninstall()
+            self.load_installation()
+            active, pid = service_state()
+            if not active or pid <= 0:
+                raise Deferred("Agent service is not running; automatic updates wait until it is started")
+            previous = read_bytes(AGENT_EXECUTABLE, MAX_SOURCE)
+            uninstall = read_bytes(UNINSTALL_EXECUTABLE, MAX_SOURCE)
+            if source_version(previous) != self.current_version:
+                raise Deferred("The installed agent changed while checking for updates")
+            source_version(uninstall)
+            self.ensure_directory()
+            atomic_bytes(UPDATE_DIR / "agent.backup.py", previous)
+            atomic_bytes(UPDATE_DIR / "uninstall.backup.py", uninstall)
+            atomic_bytes(UPDATE_DIR / "candidate.py", source)
+            journal = {"id": uuid.uuid4().hex, "phase": "installing", "installation": self.installation, "previousVersion": self.current_version, "targetVersion": self.target["version"], "targetSha256": self.target["sha256"], "agentBackupSha256": sha256(previous), "uninstallBackupSha256": sha256(uninstall), "startedAt": time.time()}
+            self.save_journal(journal)
+            self.status("updating")
+            self.check_uninstall()
+            self.load_installation()
+            systemctl("stop", AGENT_SERVICE)
+            atomic_bytes(AGENT_EXECUTABLE, source, 0o755)
+            atomic_bytes(UNINSTALL_EXECUTABLE, source, 0o755)
+            self.save_journal(journal, phase="verifying", switchedAt=time.time())
+        # The candidate takes the same lock during startup credential recovery.
+        # Never wait for its start or authenticated heartbeat while holding it.
+        systemctl("start", AGENT_SERVICE)
+        if not self.wait_healthy(journal):
+            raise UpdateError("The new agent did not report a healthy authenticated heartbeat; the previous release will be restored")
+        self.check_uninstall()
+        self.load_installation()
+        if any(sha256(read_bytes(path, MAX_SOURCE)) != journal["targetSha256"] for path in (AGENT_EXECUTABLE, UNINSTALL_EXECUTABLE)):
+            raise UpdateError("Installed agent code changed before update verification finished")
+        self.status("updated", current=journal["targetVersion"])
+        self.current_version = journal["targetVersion"]
+        self.finish_journal()
+        return "updated"
+
+    def wait_healthy(self, journal):
+        deadline = time.monotonic() + HEALTH_TIMEOUT
+        while time.monotonic() < deadline:
+            self.check_uninstall()
+            try:
+                active, pid = service_state()
+                health = read_json(AGENT_HEALTH, 4096)
+                checked = health.get("checkedAt")
+                if active and pid > 0 and type(health.get("pid")) is int and health["pid"] == pid and health.get("version") == journal["targetVersion"] and health.get("updateId") == journal["id"] and type(checked) in (float, int) and math.isfinite(checked) and journal["switchedAt"] <= checked <= time.time() + 5:
+                    return True
+            except (UpdateError, OSError, ValueError):
+                pass
+            time.sleep(2)
+        return False
+
+    def rollback(self, journal, error):
+        self.target = {"version": journal["targetVersion"], "sha256": journal["targetSha256"]}
+        self.check_uninstall()
+        self.load_installation()
+        if journal["installation"] != self.installation:
+            raise UpdateError("Agent enrollment changed; rollback needs operator review")
+        backups = [(AGENT_EXECUTABLE, "agent.backup.py", "agentBackupSha256"), (UNINSTALL_EXECUTABLE, "uninstall.backup.py", "uninstallBackupSha256")]
+        sources = []
+        for destination, name, checksum in backups:
+            source = read_bytes(UPDATE_DIR / name, MAX_SOURCE)
+            if sha256(source) != journal[checksum]:
+                raise UpdateError("Agent rollback backup failed integrity verification; repair the installation")
+            source_version(source)
+            sources.append((destination, source))
+        if source_version(sources[0][1]) != journal["previousVersion"]:
+            raise UpdateError("Agent rollback version does not match its recovery journal")
+        self.save_journal(journal, phase="rollback")
+        with self.activity_lock() as activity:
+            if activity is None:
+                raise Deferred("Agent rollback is waiting for the active operation to finish")
+            self.check_uninstall()
+            self.load_installation()
+            systemctl("stop", AGENT_SERVICE)
+            for destination, source in sources:
+                atomic_bytes(destination, source, 0o755)
+            self.save_journal(journal, rollbackSwitchedAt=time.time())
+        systemctl("start", AGENT_SERVICE)
+        rollback_health = {**journal, "targetVersion": journal["previousVersion"], "switchedAt": journal["rollbackSwitchedAt"]}
+        if not self.wait_healthy(rollback_health):
+            raise UpdateError("The previous agent release was restored but its authenticated health is unconfirmed; recovery will retry")
+        self.hold(self.target, error)
+        self.status("failed", error, current=journal["previousVersion"])
+        self.current_version = journal["previousVersion"]
+        self.finish_journal()
+        return "failed"
+
+    def run(self):
+        with exclusive_lock(UPDATE_LOCK) as updater:
+            if updater is None:
+                return "deferred"  # Preserve the active worker's status.
+            recovering = False
+            try:
+                self.check_uninstall()
+                self.load_installation()
+                if UPDATE_JOURNAL.exists() or UPDATE_JOURNAL.is_symlink():
+                    journal = self.read_journal()
+                    self.current_version = journal["previousVersion"]
+                    recovering = True
+                    return self.rollback(journal, "An interrupted agent update was rolled back to the previous release")
+                self.current_version = source_version(read_bytes(AGENT_EXECUTABLE, MAX_SOURCE))
+                self.status("checking")
+                self.target = self.manifest()
+                if version(self.target["version"]) <= version(self.current_version):
+                    return self.status("current")
+                if self.release_held():
+                    return self.status("failed", "This release previously failed verification and is held until the controller publishes a different release")
+                source = self.fetch("/api/agent/release/ec-agent.py", MAX_SOURCE)
+                if sha256(source) != self.target["sha256"]:
+                    raise IntegrityError("The agent release checksum does not match its manifest")
+                if source_version(source) != self.target["version"]:
+                    raise IntegrityError("The agent source VERSION does not match its release manifest")
+                return self.install(source)
+            except Deferred as error:
+                return self.status("deferred", error)
+            except ContactError as error:
+                return self.status("deferred", error)
+            except (UpdateError, OSError, ValueError, KeyError) as error:
+                message = str(error) if isinstance(error, UpdateError) else "Agent update could not finish; inspect the updater service"
+                if UPDATE_JOURNAL.exists():
+                    if recovering:
+                        return self.status("failed", "Agent update recovery is pending; repair the updater service before accepting new operations")
+                    try:
+                        return self.rollback(self.read_journal(), message)
+                    except Deferred as deferred:
+                        return self.status("deferred", deferred)
+                    except (UpdateError, OSError, ValueError, KeyError):
+                        return self.status("failed", "Agent update recovery is pending; repair the updater service before accepting new operations")
+                if isinstance(error, IntegrityError) and self.target:
+                    self.hold(self.target, message)
+                return self.status("failed", message)
+
+
+def main():
+    if os.geteuid() != 0:
+        raise UpdateError("Run the agent updater through its installed root service")
+    Updater().run()
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (UpdateError, OSError, ValueError, KeyError):
+        print("Electric Cherry agent updater failed; inspect its installation and update status", file=sys.stderr)
+        sys.exit(1)

static/agent/ec-agent.py

@@ -29,7 +29,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.1.0"
+VERSION = "1.2.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
@@ -39,6 +39,13 @@ AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service")
 UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
 UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service")
 UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
+UPDATE_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-update.py")
+UPDATE_UNIT = Path("/etc/systemd/system/electric-cherry-update.service")
+UPDATE_TIMER = Path("/etc/systemd/system/electric-cherry-update.timer")
+UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-cherry-update.timer")
+UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
+UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
+AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
 ROOT_MARKER = ".electric-cherry.json"
 
 
@@ -105,7 +112,7 @@ def uninstall_root(value, identity=None):
     forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"}
     if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden:
         raise AgentError("Host data directory is unsafe to remove")
-    for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT):
+    for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT, UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER, AGENT_EXECUTABLE.with_suffix(".py.update"), UNINSTALL_EXECUTABLE.with_suffix(".py.update")):
         if installed == root or root in installed.parents:
             raise AgentError("Host data directory overlaps the agent installation")
     protected_path(root, missing=True)
@@ -499,6 +506,19 @@ class Uninstaller:
 
     def remove_data(self):
         protected_path(AGENT_CONFIG.parent, missing=True)
+        # Prevent a scheduled update from recreating agent files during removal.
+        # Stopping the worker is safe here: the installation and any backups are
+        # being removed together, and the durable decommission request blocks
+        # future update attempts.
+        if not self.job.get("updaterStopped"):
+            for unit in (UPDATE_TIMER, UPDATE_UNIT):
+                protected_file(unit, missing=True)
+                if unit.exists():
+                    if unit == UPDATE_TIMER:
+                        systemctl("disable", "--now", unit.name)
+                    else:
+                        systemctl("stop", unit.name)
+            self.save(updaterStopped=True)
         if not self.job.get("agentStopped"):
             systemctl("disable", "--now", AGENT_UNIT.name)
             self.save(agentStopped=True)
@@ -512,7 +532,7 @@ class Uninstaller:
                 raise AgentError("This platform cannot safely remove managed host files")
             shutil.rmtree(root)
             sync_parent(root)
-        for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT):
+        for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER, AGENT_EXECUTABLE.with_suffix(".py.update"), UNINSTALL_EXECUTABLE.with_suffix(".py.update")):
             protected_file(path, missing=True)
             path.unlink(missing_ok=True)
             sync_parent(path)
@@ -591,6 +611,74 @@ class Agent:
         except (AgentError, OSError, ValueError):
             return False
 
+    def can_update(self):
+        try:
+            if Path(self.config_path) != AGENT_CONFIG:
+                return False
+            uninstall_ownership(self.config)
+            for path in (UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER):
+                protected_file(path)
+            protected_path(UPDATE_TIMER_ENABLED.parent)
+            return UPDATE_TIMER_ENABLED.is_symlink() and UPDATE_TIMER_ENABLED.resolve() == UPDATE_TIMER
+        except (AgentError, OSError, ValueError):
+            return False
+
+    def update_inventory(self):
+        result = {"enabled": self.can_update(), "state": "current", "currentVersion": VERSION}
+        if Path(self.config_path) != AGENT_CONFIG:
+            return result
+        try:
+            protected_file(UPDATE_STATUS, missing=True)
+            if not UPDATE_STATUS.exists():
+                return result
+            if UPDATE_STATUS.stat().st_size > 16384:
+                raise ValueError("Invalid update status")
+            saved = json.loads(UPDATE_STATUS.read_text())
+            if saved.get("state") in ("current", "checking", "deferred", "updating", "updated", "failed"):
+                result["state"] = saved["state"]
+            for key in ("targetVersion", "error"):
+                if isinstance(saved.get(key), str):
+                    result[key] = saved[key][:2000 if key == "error" else 80]
+            checked_at = saved.get("checkedAt")
+            if isinstance(checked_at, (int, float)) and not isinstance(checked_at, bool):
+                result["checkedAt"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(checked_at))
+            elif isinstance(checked_at, str):
+                result["checkedAt"] = checked_at[:80]
+        except (AgentError, OSError, ValueError, TypeError, AttributeError, OverflowError):
+            result.update(state="failed", error="Agent update status is unavailable; inspect the update service.")
+        return result
+
+    def update_pending(self):
+        # A persisted transaction survives crashes and is cleared only by the
+        # independent worker after health confirmation or a completed rollback.
+        return Path(self.config_path) == AGENT_CONFIG and (UPDATE_JOURNAL.exists() or UPDATE_JOURNAL.is_symlink())
+
+    @contextlib.contextmanager
+    def activity(self):
+        path = self.root / "agent" / "activity.lock"
+        descriptor = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+        try:
+            try:
+                fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError:
+                yield False
+                return
+            yield True
+        finally:
+            os.close(descriptor)
+
+    def record_health(self):
+        if Path(self.config_path) != AGENT_CONFIG or not self.can_update():
+            return
+        value = {"version": VERSION, "pid": os.getpid(), "checkedAt": time.time()}
+        protected_file(AGENT_HEALTH, missing=True)
+        if self.update_pending():
+            protected_file(UPDATE_JOURNAL)
+            journal = json.loads(UPDATE_JOURNAL.read_text())
+            if isinstance(journal.get("id"), str):
+                value["updateId"] = journal["id"]
+        atomic_json(AGENT_HEALTH, value)
+
     def decommission(self, operation):
         payload = operation.get("payload", {})
         if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]:
@@ -733,7 +821,9 @@ class Agent:
             for line in Path("/etc/os-release").read_text().splitlines():
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
-        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
+        update = self.update_inventory()
+        capabilities = (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
         health = self.inspect(instance_id, generation)
@@ -1462,40 +1552,55 @@ class Agent:
         host = response.get("host", {})
         if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int):
             self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]}
+        # Written only after the controller accepted this process's authenticated
+        # heartbeat. Service liveness alone cannot prove an update works.
+        self.record_health()
 
     def heartbeat_loop(self):
         while not self.stopping.wait(15):
             try:
                 self.heartbeat()
-            except AgentError:
+            except (AgentError, OSError, ValueError):
                 pass
 
+    def poll_operations(self):
+        with self.activity() as acquired:
+            if not acquired or self.stopping.is_set() or self.update_pending():
+                return
+            if self.config.get("pendingCredential"):
+                self.recover_credential()
+            response = self.request("GET", "/api/agent/operations")
+            for operation in response.get("operations", []):
+                if self.stopping.is_set():
+                    break
+                self.current_operation = operation["id"]
+                try:
+                    accepted, _ = self.journal.accept(operation)
+                    self.report(accepted)
+                    self.report(self.execute(operation, report_running=True))
+                finally:
+                    self.current_operation = None
+
     def run(self):
-        if self.config.get("pendingCredential"):
-            self.recover_credential()
         # Obtain the configured host port range before accepting any deployment.
         # Enrollment's default values must never race the first heartbeat.
         while not self.stopping.is_set():
             try:
+                with self.activity() as acquired:
+                    if not acquired:
+                        self.stopping.wait(3)
+                        continue
+                    if self.config.get("pendingCredential"):
+                        self.recover_credential()
                 self.heartbeat()
                 break
-            except AgentError:
+            except (AgentError, OSError, ValueError):
                 self.stopping.wait(3)
         heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
         heartbeats.start()
         while not self.stopping.is_set():
             try:
-                if self.config.get("pendingCredential"):
-                    self.recover_credential()
-                response = self.request("GET", "/api/agent/operations")
-                for operation in response.get("operations", []):
-                    if self.stopping.is_set():
-                        break
-                    self.current_operation = operation["id"]
-                    accepted, _ = self.journal.accept(operation)
-                    self.report(accepted)
-                    self.report(self.execute(operation, report_running=True))
-                    self.current_operation = None
+                self.poll_operations()
             except (AgentError, OSError, ValueError, KeyError):
                 # Logs deliberately omit URLs, tokens, payloads and raw errors.
                 print("Host controller contact unavailable; existing containers continue running", file=sys.stderr)

static/agent/install.sh

@@ -8,6 +8,8 @@ controller=''
 enrollment_token=''
 data_dir='/var/lib/electric-cherry'
 reenroll=''
+existing_enrollment=false
+existing_data_root=false
 while [ "$#" -gt 0 ]; do
     case "$1" in
         --controller) controller="${2:?Missing controller URL}"; shift 2 ;;
@@ -31,7 +33,16 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; }
 
 export DEBIAN_FRONTEND=noninteractive
 apt-get update -qq
-apt-get install -y --no-install-recommends ca-certificates curl python3
+apt-get install -y --no-install-recommends ca-certificates curl python3 util-linux
+installation_preflight() {
+    if [ "$existing_enrollment" = true ] && [ ! -f /etc/electric-cherry/agent.json ]; then
+        echo 'The existing enrollment disappeared during installation. Stop and inspect host deletion before trying again.' >&2
+        return 1
+    fi
+    if [ "$existing_data_root" = true ] && [ ! -d "$data_dir" ]; then
+        echo 'The existing data directory disappeared during installation. Stop and inspect host deletion before trying again.' >&2
+        return 1
+    fi
 python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY'
 # INSTALLATION_PREFLIGHT: this runs before touching managed directories or services.
 import json, pathlib, re, stat, sys
@@ -51,9 +62,14 @@ if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root)
     raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.')
 installed_paths = [config,
     pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-agent.py.update'),
     pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py.update'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-update.py'),
     pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'),
-    pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service')]
+    pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-update.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-update.timer')]
 if any(root == path or root in path.parents for path in installed_paths):
     raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.')
 
@@ -66,10 +82,13 @@ def trusted_directories(path):
         if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
             raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.')
 
-def trusted_json(path):
+def trusted_file(path):
     info = path.lstat()
     if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
         raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.')
+
+def trusted_json(path):
+    trusted_file(path)
     value = json.loads(path.read_text())
     if not isinstance(value, dict):
         raise SystemExit('Existing installation metadata is invalid.')
@@ -89,6 +108,13 @@ if mountinfo.exists():
 request = config.with_name('decommission.json')
 if request.exists() or request.is_symlink():
     raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.')
+trusted_directories(config.parent / 'update')
+journal = config.parent / 'update' / 'journal.json'
+if journal.exists() or journal.is_symlink():
+    raise SystemExit('An automatic update needs recovery. Let electric-cherry-update.service finish or repair that update before rerunning the installer.')
+update_lock = config.with_name('update.lock')
+if update_lock.exists() or update_lock.is_symlink():
+    trusted_file(update_lock)
 saved = None
 if config.exists() or config.is_symlink():
     saved = trusted_json(config)
@@ -108,6 +134,62 @@ elif root.exists() and any(root.iterdir()):
         raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.')
 for directory in ('agent', 'instances', 'libraries'):
     trusted_directories(root / directory)
+activity_lock = root / 'agent' / 'activity.lock'
+if activity_lock.exists() or activity_lock.is_symlink():
+    trusted_file(activity_lock)
+PY
+}
+if [ -f /etc/electric-cherry/agent.json ]; then existing_enrollment=true; fi
+if [ -d "$data_dir" ]; then existing_data_root=true; fi
+installation_preflight
+temporary_dir=$(mktemp -d)
+updater_timer_was_active=false
+update_lock_held=false
+activity_lock_held=false
+installer_finished=false
+installation_cleanup() {
+    local ec_install_status=$?
+    trap - EXIT
+    if [ "$activity_lock_held" = true ]; then
+        flock --unlock 8 || true
+        exec 8>&-
+    fi
+    if [ "$update_lock_held" = true ]; then
+        flock --unlock 9 || true
+        exec 9>&-
+    fi
+    # Recovery must remain scheduled when an upgrade or its second preflight
+    # fails. Release the worker lock before restarting the prior active timer.
+    if [ "$installer_finished" != true ] && [ "$updater_timer_was_active" = true ]; then
+        systemctl start electric-cherry-update.timer >/dev/null 2>&1 || true
+    fi
+    rm -rf "$temporary_dir" || true
+    exit "$ec_install_status"
+}
+trap installation_cleanup EXIT
+controller="${controller%/}"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent-update.py" -o "$temporary_dir/ec-agent-update.py"
+python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" "$temporary_dir/ec-agent-update.py" <<'PY'
+# INSTALLATION_RELEASE: check both downloads before installing or executing either.
+import ast, hashlib, json, pathlib, re, sys
+manifest = json.loads(pathlib.Path(sys.argv[1]).read_text())
+agent = pathlib.Path(sys.argv[2]).read_bytes()
+updater = pathlib.Path(sys.argv[3]).read_bytes()
+if hashlib.sha256(agent).hexdigest() != manifest.get('sha256'):
+    raise SystemExit('Agent release checksum verification failed.')
+if hashlib.sha256(updater).hexdigest() != manifest.get('updaterSha256'):
+    raise SystemExit('Updater release checksum verification failed.')
+if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1 or manifest.get('updaterProtocolVersion') != 1:
+    raise SystemExit('Unsupported agent or updater release manifest.')
+versions = [node.value.value for node in ast.parse(agent).body
+            if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == 'VERSION' for target in node.targets)
+            and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str)]
+if len(versions) != 1 or versions[0] != manifest.get('version') or not re.fullmatch(r'(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)', versions[0]):
+    raise SystemExit('Agent release version does not match its manifest.')
+compile(agent, 'ec-agent.py', 'exec')
+compile(updater, 'ec-agent-update.py', 'exec')
 PY
 if ! command -v docker >/dev/null; then
     if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then
@@ -118,29 +200,37 @@ if ! command -v docker >/dev/null; then
     systemctl enable --now docker
 fi
 docker info >/dev/null 2>&1 || { echo 'Docker is installed but unavailable. Start or repair it, then rerun.' >&2; exit 1; }
-install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances"
-install -d -m 0700 "$data_dir/agent" /etc/electric-cherry
-available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}')
-[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; }
-temporary_dir=$(mktemp -d)
-trap 'rm -rf "$temporary_dir"' EXIT
-controller="${controller%/}"
-curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json"
-curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py"
-python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" <<'PY'
-import hashlib, json, pathlib, sys
-manifest = json.loads(pathlib.Path(sys.argv[1]).read_text())
-actual = hashlib.sha256(pathlib.Path(sys.argv[2]).read_bytes()).hexdigest()
-if actual != manifest.get('sha256'):
-    raise SystemExit('Agent release checksum verification failed.')
-if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1:
-    raise SystemExit('Unsupported agent release manifest.')
-PY
+install -d -m 0700 /etc/electric-cherry
+# Wait for an already-running updater to finish; never interrupt its rollback.
+# The same lock also prevents a timer firing while files are being installed.
+if systemctl cat electric-cherry-update.timer >/dev/null 2>&1; then
+    if systemctl is-active --quiet electric-cherry-update.timer; then
+        updater_timer_was_active=true
+    fi
+    systemctl stop electric-cherry-update.timer
+fi
+exec 9>/etc/electric-cherry/update.lock
+flock --exclusive 9
+update_lock_held=true
+installation_preflight
+# Serialize with running operations, especially a deletion request that could
+# otherwise start between preflight and stopping the main agent.
+if [ -d "$data_dir/agent" ]; then
+    exec 8>"$data_dir/agent/activity.lock"
+    flock --exclusive 8
+    activity_lock_held=true
+    installation_preflight
+fi
 # Stop only the agent while replacing its own executable/journal owner. Existing
 # game containers keep running under Docker throughout installation and updates.
 if systemctl cat electric-cherry-agent.service >/dev/null 2>&1; then
     systemctl stop electric-cherry-agent.service
 fi
+installation_preflight
+install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances"
+install -d -m 0700 "$data_dir/agent"
+available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}')
+[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; }
 install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py 2>/dev/null || {
     install -d -m 0755 /usr/local/libexec
     install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py
@@ -148,6 +238,7 @@ install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-
 # A separate worker survives stopping/removing the agent service. Its only
 # entry point is a fixed, authenticated deletion request prepared by the agent.
 install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py
+install -m 0755 "$temporary_dir/ec-agent-update.py" /usr/local/libexec/electric-cherry-update.py
 printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token"
 unset enrollment_token
 python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"}
@@ -215,8 +306,41 @@ UMask=0077
 [Install]
 WantedBy=multi-user.target
 EOF
+cat > /etc/systemd/system/electric-cherry-update.service <<'EOF'
+[Unit]
+Description=Update the Electric Cherry Host Agent
+After=network-online.target docker.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-update.py
+TimeoutStartSec=0
+UMask=0077
+EOF
+cat > /etc/systemd/system/electric-cherry-update.timer <<'EOF'
+[Unit]
+Description=Check for Electric Cherry Host Agent Updates
+
+[Timer]
+OnBootSec=2min
+OnUnitInactiveSec=5min
+RandomizedDelaySec=30s
+AccuracySec=1s
+Unit=electric-cherry-update.service
+
+[Install]
+WantedBy=timers.target
+EOF
 systemctl daemon-reload
 systemctl enable electric-cherry-agent
+if [ "$activity_lock_held" = true ]; then
+    flock --unlock 8
+    exec 8>&-
+    activity_lock_held=false
+fi
 systemctl restart electric-cherry-agent
 systemctl is-active --quiet electric-cherry-agent || { echo 'Agent service did not start. Inspect journalctl -u electric-cherry-agent.' >&2; exit 1; }
-echo 'Electric Cherry host agent installed and enrolled. Host readiness is shown in Hosts.'
+systemctl enable --now electric-cherry-update.timer
+installer_finished=true
+echo 'Electric Cherry host agent installed and enrolled. Automatic agent updates are enabled; host readiness is shown in Hosts.'

static/agent/manifest.json

@@ -1,6 +1,8 @@
 {
-  "version": "1.1.0",
+  "version": "1.2.0",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "67046a3d81f251e148f2323cade255ff584a622750a8c05ee27f96c70e09fcc7"
+  "sha256": "c0c08bc50daf09f4e6e141d4119cee68973b47c29845b2f3620e4579d9cd7daa",
+  "updaterProtocolVersion": 1,
+  "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }