Add automatic agent updates with verified rollback

agent/ec_agent.py

@@ -29,7 +29,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.1.0"
+VERSION = "1.2.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
@@ -39,6 +39,13 @@ AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service")
 UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
 UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service")
 UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
+UPDATE_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-update.py")
+UPDATE_UNIT = Path("/etc/systemd/system/electric-cherry-update.service")
+UPDATE_TIMER = Path("/etc/systemd/system/electric-cherry-update.timer")
+UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-cherry-update.timer")
+UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
+UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
+AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
 ROOT_MARKER = ".electric-cherry.json"
 
 
@@ -105,7 +112,7 @@ def uninstall_root(value, identity=None):
     forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"}
     if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden:
         raise AgentError("Host data directory is unsafe to remove")
-    for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT):
+    for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT, UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER, AGENT_EXECUTABLE.with_suffix(".py.update"), UNINSTALL_EXECUTABLE.with_suffix(".py.update")):
         if installed == root or root in installed.parents:
             raise AgentError("Host data directory overlaps the agent installation")
     protected_path(root, missing=True)
@@ -499,6 +506,19 @@ class Uninstaller:
 
     def remove_data(self):
         protected_path(AGENT_CONFIG.parent, missing=True)
+        # Prevent a scheduled update from recreating agent files during removal.
+        # Stopping the worker is safe here: the installation and any backups are
+        # being removed together, and the durable decommission request blocks
+        # future update attempts.
+        if not self.job.get("updaterStopped"):
+            for unit in (UPDATE_TIMER, UPDATE_UNIT):
+                protected_file(unit, missing=True)
+                if unit.exists():
+                    if unit == UPDATE_TIMER:
+                        systemctl("disable", "--now", unit.name)
+                    else:
+                        systemctl("stop", unit.name)
+            self.save(updaterStopped=True)
         if not self.job.get("agentStopped"):
             systemctl("disable", "--now", AGENT_UNIT.name)
             self.save(agentStopped=True)
@@ -512,7 +532,7 @@ class Uninstaller:
                 raise AgentError("This platform cannot safely remove managed host files")
             shutil.rmtree(root)
             sync_parent(root)
-        for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT):
+        for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER, AGENT_EXECUTABLE.with_suffix(".py.update"), UNINSTALL_EXECUTABLE.with_suffix(".py.update")):
             protected_file(path, missing=True)
             path.unlink(missing_ok=True)
             sync_parent(path)
@@ -591,6 +611,74 @@ class Agent:
         except (AgentError, OSError, ValueError):
             return False
 
+    def can_update(self):
+        try:
+            if Path(self.config_path) != AGENT_CONFIG:
+                return False
+            uninstall_ownership(self.config)
+            for path in (UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER):
+                protected_file(path)
+            protected_path(UPDATE_TIMER_ENABLED.parent)
+            return UPDATE_TIMER_ENABLED.is_symlink() and UPDATE_TIMER_ENABLED.resolve() == UPDATE_TIMER
+        except (AgentError, OSError, ValueError):
+            return False
+
+    def update_inventory(self):
+        result = {"enabled": self.can_update(), "state": "current", "currentVersion": VERSION}
+        if Path(self.config_path) != AGENT_CONFIG:
+            return result
+        try:
+            protected_file(UPDATE_STATUS, missing=True)
+            if not UPDATE_STATUS.exists():
+                return result
+            if UPDATE_STATUS.stat().st_size > 16384:
+                raise ValueError("Invalid update status")
+            saved = json.loads(UPDATE_STATUS.read_text())
+            if saved.get("state") in ("current", "checking", "deferred", "updating", "updated", "failed"):
+                result["state"] = saved["state"]
+            for key in ("targetVersion", "error"):
+                if isinstance(saved.get(key), str):
+                    result[key] = saved[key][:2000 if key == "error" else 80]
+            checked_at = saved.get("checkedAt")
+            if isinstance(checked_at, (int, float)) and not isinstance(checked_at, bool):
+                result["checkedAt"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(checked_at))
+            elif isinstance(checked_at, str):
+                result["checkedAt"] = checked_at[:80]
+        except (AgentError, OSError, ValueError, TypeError, AttributeError, OverflowError):
+            result.update(state="failed", error="Agent update status is unavailable; inspect the update service.")
+        return result
+
+    def update_pending(self):
+        # A persisted transaction survives crashes and is cleared only by the
+        # independent worker after health confirmation or a completed rollback.
+        return Path(self.config_path) == AGENT_CONFIG and (UPDATE_JOURNAL.exists() or UPDATE_JOURNAL.is_symlink())
+
+    @contextlib.contextmanager
+    def activity(self):
+        path = self.root / "agent" / "activity.lock"
+        descriptor = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+        try:
+            try:
+                fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError:
+                yield False
+                return
+            yield True
+        finally:
+            os.close(descriptor)
+
+    def record_health(self):
+        if Path(self.config_path) != AGENT_CONFIG or not self.can_update():
+            return
+        value = {"version": VERSION, "pid": os.getpid(), "checkedAt": time.time()}
+        protected_file(AGENT_HEALTH, missing=True)
+        if self.update_pending():
+            protected_file(UPDATE_JOURNAL)
+            journal = json.loads(UPDATE_JOURNAL.read_text())
+            if isinstance(journal.get("id"), str):
+                value["updateId"] = journal["id"]
+        atomic_json(AGENT_HEALTH, value)
+
     def decommission(self, operation):
         payload = operation.get("payload", {})
         if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]:
@@ -733,7 +821,9 @@ class Agent:
             for line in Path("/etc/os-release").read_text().splitlines():
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
-        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
+        update = self.update_inventory()
+        capabilities = (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
         health = self.inspect(instance_id, generation)
@@ -1462,40 +1552,55 @@ class Agent:
         host = response.get("host", {})
         if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int):
             self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]}
+        # Written only after the controller accepted this process's authenticated
+        # heartbeat. Service liveness alone cannot prove an update works.
+        self.record_health()
 
     def heartbeat_loop(self):
         while not self.stopping.wait(15):
             try:
                 self.heartbeat()
-            except AgentError:
+            except (AgentError, OSError, ValueError):
                 pass
 
+    def poll_operations(self):
+        with self.activity() as acquired:
+            if not acquired or self.stopping.is_set() or self.update_pending():
+                return
+            if self.config.get("pendingCredential"):
+                self.recover_credential()
+            response = self.request("GET", "/api/agent/operations")
+            for operation in response.get("operations", []):
+                if self.stopping.is_set():
+                    break
+                self.current_operation = operation["id"]
+                try:
+                    accepted, _ = self.journal.accept(operation)
+                    self.report(accepted)
+                    self.report(self.execute(operation, report_running=True))
+                finally:
+                    self.current_operation = None
+
     def run(self):
-        if self.config.get("pendingCredential"):
-            self.recover_credential()
         # Obtain the configured host port range before accepting any deployment.
         # Enrollment's default values must never race the first heartbeat.
         while not self.stopping.is_set():
             try:
+                with self.activity() as acquired:
+                    if not acquired:
+                        self.stopping.wait(3)
+                        continue
+                    if self.config.get("pendingCredential"):
+                        self.recover_credential()
                 self.heartbeat()
                 break
-            except AgentError:
+            except (AgentError, OSError, ValueError):
                 self.stopping.wait(3)
         heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
         heartbeats.start()
         while not self.stopping.is_set():
             try:
-                if self.config.get("pendingCredential"):
-                    self.recover_credential()
-                response = self.request("GET", "/api/agent/operations")
-                for operation in response.get("operations", []):
-                    if self.stopping.is_set():
-                        break
-                    self.current_operation = operation["id"]
-                    accepted, _ = self.journal.accept(operation)
-                    self.report(accepted)
-                    self.report(self.execute(operation, report_running=True))
-                    self.current_operation = None
+                self.poll_operations()
             except (AgentError, OSError, ValueError, KeyError):
                 # Logs deliberately omit URLs, tokens, payloads and raw errors.
                 print("Host controller contact unavailable; existing containers continue running", file=sys.stderr)

agent/ec_agent_update.py

@@ -0,0 +1,479 @@
+#!/usr/bin/env python3
+"""Stable Electric Cherry agent updater; only verified agent code is replaced.
+
+The installer owns this bootstrap and its systemd timer. Updates never execute
+downloaded code before installing it, alter game containers, or edit enrollment
+credentials. A durable journal makes an interrupted switch roll back safely.
+"""
+
+import ast
+import contextlib
+import fcntl
+import hashlib
+import http.client
+import json
+import math
+import os
+from pathlib import Path
+import re
+import stat
+import subprocess
+import sys
+import time
+import urllib.parse
+import urllib.request
+import uuid
+
+UPDATER_PROTOCOL_VERSION = 1
+AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
+AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
+UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
+UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
+UPDATE_LOCK = Path("/etc/electric-cherry/update.lock")
+UPDATE_DIR = Path("/etc/electric-cherry/update")
+UPDATE_JOURNAL = UPDATE_DIR / "journal.json"
+UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
+AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
+AGENT_SERVICE = "electric-cherry-agent.service"
+MAX_MANIFEST = 16 * 1024
+MAX_SOURCE = 8 * 1024 * 1024
+HEALTH_TIMEOUT = 90
+VERSION_RE = re.compile(r"^(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})$")
+DIGEST_RE = re.compile(r"^[0-9a-f]{64}$")
+
+
+class UpdateError(Exception):
+    pass
+
+
+class IntegrityError(UpdateError):
+    pass
+
+
+class ContactError(UpdateError):
+    pass
+
+
+class Deferred(UpdateError):
+    pass
+
+
+def trusted(path, missing=False, directory=False):
+    path = Path(path)
+    for current in (*reversed(path.parents), path):
+        try:
+            info = current.lstat()
+        except FileNotFoundError:
+            if missing:
+                continue
+            raise UpdateError("Agent update files are unavailable; repair the installation") from None
+        expected_type = stat.S_ISDIR if current != path or directory else stat.S_ISREG
+        if not expected_type(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+            raise UpdateError("Agent update paths must be regular root-owned files and directories that other users cannot modify")
+    return path
+
+
+def sync_parent(path):
+    descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def atomic_bytes(path, value, mode=0o600):
+    path = trusted(path, missing=True)
+    trusted(path.parent, directory=True)
+    # Code staging uses known names so recovery and host removal can also
+    # remove a file left behind by a power loss during the atomic write.
+    temporary = path.with_name(path.name + ".update") if path in (AGENT_EXECUTABLE, UNINSTALL_EXECUTABLE) else path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp")
+    try:
+        if temporary.exists() or temporary.is_symlink():
+            trusted(temporary)
+            temporary.unlink()
+            sync_parent(temporary)
+        descriptor = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, mode)
+        with os.fdopen(descriptor, "wb") as stream:
+            stream.write(value)
+            stream.flush()
+            os.fsync(stream.fileno())
+        trusted(path, missing=True)
+        os.replace(temporary, path)
+        sync_parent(path)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def atomic_json(path, value):
+    atomic_bytes(path, json.dumps(value, sort_keys=True, allow_nan=False).encode())
+
+
+def read_bytes(path, limit):
+    trusted(path)
+    with open(path, "rb") as stream:
+        value = stream.read(limit + 1)
+    if len(value) > limit:
+        raise UpdateError("An agent update file exceeds its size limit")
+    return value
+
+
+def read_json(path, limit=MAX_MANIFEST):
+    try:
+        value = json.loads(read_bytes(path, limit))
+    except (ValueError, UnicodeError):
+        raise UpdateError("Agent update metadata is invalid") from None
+    if not isinstance(value, dict):
+        raise UpdateError("Agent update metadata is invalid")
+    return value
+
+
+def sha256(value):
+    return hashlib.sha256(value).hexdigest()
+
+
+def version(value):
+    if not isinstance(value, str) or not VERSION_RE.fullmatch(value):
+        raise IntegrityError("Agent releases must use a numeric major.minor.patch version")
+    return tuple(int(part) for part in value.split("."))
+
+
+def source_version(source):
+    try:
+        tree = ast.parse(source, filename="electric-cherry-agent.py")
+        compile(tree, "electric-cherry-agent.py", "exec")
+    except (SyntaxError, ValueError, TypeError, RecursionError, MemoryError):
+        raise IntegrityError("The agent release is not valid Python source") from None
+    declarations = [node.value.value for node in tree.body if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == "VERSION" for target in node.targets) and isinstance(node.value, ast.Constant)]
+    if len(declarations) != 1:
+        raise IntegrityError("The agent release must declare one explicit VERSION")
+    version(declarations[0])
+    return declarations[0]
+
+
+@contextlib.contextmanager
+def exclusive_lock(path):
+    trusted(path, missing=True)
+    trusted(Path(path).parent, directory=True)
+    descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
+    with os.fdopen(descriptor, "r+") as stream:
+        try:
+            fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            yield None
+            return
+        try:
+            yield stream
+        finally:
+            fcntl.flock(stream, fcntl.LOCK_UN)
+
+
+def systemctl(*args):
+    try:
+        result = subprocess.run(["systemctl", *args], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=60, check=False)
+    except (OSError, subprocess.TimeoutExpired):
+        raise UpdateError("Agent service management is unavailable") from None
+    if result.returncode:
+        raise UpdateError("Agent service management did not complete")
+    return result.stdout.decode("utf-8", "replace")
+
+
+def service_state():
+    output = systemctl("show", AGENT_SERVICE, "--property=ActiveState", "--property=MainPID")
+    fields = dict(line.split("=", 1) for line in output.splitlines() if "=" in line)
+    try:
+        return fields.get("ActiveState") == "active", int(fields.get("MainPID", "0"))
+    except ValueError:
+        return False, 0
+
+
+class Updater:
+    def __init__(self):
+        self.installation = None
+        self.current_version = None
+        self.target = None
+
+    def load_installation(self):
+        config = read_json(AGENT_CONFIG, 64 * 1024)
+        controller = config.get("controller")
+        root_value = config.get("root")
+        host_id = config.get("hostId")
+        if not isinstance(controller, str) or not isinstance(root_value, str) or not isinstance(host_id, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,127}", host_id):
+            raise UpdateError("Agent enrollment is incomplete")
+        parsed = urllib.parse.urlsplit(controller)
+        if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise UpdateError("Agent updates require an HTTPS controller without embedded credentials")
+        root = Path(root_value)
+        if not re.fullmatch(r"/[A-Za-z0-9/_.-]+", root_value) or str(root) != root_value or ".." in root.parts or len(root.parts) < 3:
+            raise UpdateError("The enrolled agent data directory is invalid")
+        trusted(root, directory=True)
+        trusted(root / "agent", directory=True)
+        marker = read_json(root / ".electric-cherry.json")
+        controller = controller.rstrip("/")
+        expected = {"version": 1, "root": root_value, "hostId": host_id, "controller": controller}
+        if marker != expected:
+            raise UpdateError("Agent data ownership does not match this enrollment")
+        info = root.stat()
+        installation = {"hostId": host_id, "controller": controller, "root": root_value, "rootIdentity": [info.st_dev, info.st_ino]}
+        if self.installation is not None and installation != self.installation:
+            raise UpdateError("Agent enrollment changed during the update; no code was replaced")
+        self.installation = installation
+        return installation
+
+    def ensure_directory(self):
+        trusted(UPDATE_DIR, missing=True, directory=True)
+        trusted(UPDATE_DIR.parent, directory=True)
+        if not UPDATE_DIR.exists():
+            UPDATE_DIR.mkdir(mode=0o700)
+            sync_parent(UPDATE_DIR)
+
+    def check_uninstall(self):
+        if UNINSTALL_REQUEST.exists() or UNINSTALL_REQUEST.is_symlink():
+            raise Deferred("Host deletion is pending; agent updates are paused")
+
+    def status(self, state, error=None, current=None):
+        value = {"state": state, "checkedAt": time.time()}
+        if current or self.current_version:
+            value["currentVersion"] = current or self.current_version
+        if self.target:
+            value["targetVersion"] = self.target["version"]
+        if error:
+            value["error"] = str(error)[:500]
+        atomic_json(UPDATE_STATUS, value)
+        return state
+
+    def fetch(self, path, limit):
+        request = urllib.request.Request(self.installation["controller"] + path, headers={"User-Agent": "electric-cherry-updater/1"})
+        class NoRedirect(urllib.request.HTTPRedirectHandler):
+            def redirect_request(self, req, fp, code, msg, headers, newurl):
+                raise ContactError("Agent release redirects are not allowed")
+        try:
+            with urllib.request.build_opener(NoRedirect()).open(request, timeout=25) as response:
+                if response.status != 200:
+                    raise ContactError("The controller did not return a complete agent release")
+                length = response.headers.get("Content-Length")
+                if length is not None and (not length.isdecimal() or int(length) > limit):
+                    raise IntegrityError("The controller release exceeds its download size limit")
+                chunks, total = [], 0
+                deadline = time.monotonic() + 120
+                while True:
+                    if time.monotonic() >= deadline:
+                        raise ContactError("The controller release download timed out")
+                    # HTTPResponse.read1 performs one socket read. read(n) can
+                    # wait indefinitely for n bytes from a trickling server.
+                    block = response.read1(min(64 * 1024, limit + 1 - total))
+                    if not block:
+                        break
+                    total += len(block)
+                    if total > limit:
+                        raise IntegrityError("The controller release exceeds its download size limit")
+                    chunks.append(block)
+                if length is not None and total != int(length):
+                    raise ContactError("The controller release download was interrupted")
+                return b"".join(chunks)
+        except (OSError, ValueError, http.client.HTTPException):
+            raise ContactError("The controller release is unavailable; the running agent was left unchanged") from None
+
+    def manifest(self):
+        try:
+            value = json.loads(self.fetch("/api/agent/release/manifest.json", MAX_MANIFEST))
+        except (ValueError, UnicodeError):
+            raise IntegrityError("The controller release manifest is invalid") from None
+        if not isinstance(value, dict) or type(value.get("protocolVersion")) is not int or value["protocolVersion"] != 1 or value.get("platform") != "debian-amd64":
+            raise IntegrityError("The controller release is not compatible with this host agent")
+        if type(value.get("updaterProtocolVersion")) is not int or value["updaterProtocolVersion"] != UPDATER_PROTOCOL_VERSION:
+            raise Deferred("This release needs a newer updater bootstrap; rerun the host installer")
+        version(value.get("version"))
+        if not isinstance(value.get("sha256"), str) or not DIGEST_RE.fullmatch(value["sha256"]):
+            raise IntegrityError("The controller release manifest has an invalid checksum")
+        return {"version": value["version"], "sha256": value["sha256"]}
+
+    def hold(self, target, error):
+        self.ensure_directory()
+        atomic_json(UPDATE_DIR / "failed-release.json", {**target, "error": error, "failedAt": time.time()})
+
+    def release_held(self):
+        path = UPDATE_DIR / "failed-release.json"
+        if not path.exists():
+            return False
+        held = read_json(path)
+        return all(held.get(key) == self.target[key] for key in ("version", "sha256"))
+
+    def save_journal(self, journal, **changes):
+        journal.update(changes)
+        atomic_json(UPDATE_JOURNAL, journal)
+
+    def read_journal(self):
+        journal = read_json(UPDATE_JOURNAL)
+        if journal.get("installation") != self.installation or journal.get("phase") not in ("installing", "verifying", "rollback") or not isinstance(journal.get("id"), str) or not re.fullmatch(r"[a-f0-9]{32}", journal["id"]):
+            raise UpdateError("Pending agent update ownership or recovery metadata is invalid")
+        version(journal.get("previousVersion"))
+        version(journal.get("targetVersion"))
+        for field in ("targetSha256", "agentBackupSha256", "uninstallBackupSha256"):
+            if not isinstance(journal.get(field), str) or not DIGEST_RE.fullmatch(journal[field]):
+                raise UpdateError("Pending agent update recovery checksums are invalid")
+        return journal
+
+    def finish_journal(self):
+        trusted(UPDATE_JOURNAL)
+        UPDATE_JOURNAL.unlink()
+        sync_parent(UPDATE_JOURNAL)
+        # Removing the journal commits the decision. Backups are disposable
+        # only afterward, so a crash never leaves recovery without its inputs.
+        for name in ("agent.backup.py", "uninstall.backup.py", "candidate.py"):
+            path = trusted(UPDATE_DIR / name, missing=True)
+            path.unlink(missing_ok=True)
+        sync_parent(UPDATE_JOURNAL)
+
+    def activity_lock(self):
+        return exclusive_lock(Path(self.installation["root"]) / "agent" / "activity.lock")
+
+    def install(self, source):
+        with self.activity_lock() as activity:
+            if activity is None:
+                raise Deferred("The agent is executing an operation; update will retry when it is idle")
+            self.check_uninstall()
+            self.load_installation()
+            active, pid = service_state()
+            if not active or pid <= 0:
+                raise Deferred("Agent service is not running; automatic updates wait until it is started")
+            previous = read_bytes(AGENT_EXECUTABLE, MAX_SOURCE)
+            uninstall = read_bytes(UNINSTALL_EXECUTABLE, MAX_SOURCE)
+            if source_version(previous) != self.current_version:
+                raise Deferred("The installed agent changed while checking for updates")
+            source_version(uninstall)
+            self.ensure_directory()
+            atomic_bytes(UPDATE_DIR / "agent.backup.py", previous)
+            atomic_bytes(UPDATE_DIR / "uninstall.backup.py", uninstall)
+            atomic_bytes(UPDATE_DIR / "candidate.py", source)
+            journal = {"id": uuid.uuid4().hex, "phase": "installing", "installation": self.installation, "previousVersion": self.current_version, "targetVersion": self.target["version"], "targetSha256": self.target["sha256"], "agentBackupSha256": sha256(previous), "uninstallBackupSha256": sha256(uninstall), "startedAt": time.time()}
+            self.save_journal(journal)
+            self.status("updating")
+            self.check_uninstall()
+            self.load_installation()
+            systemctl("stop", AGENT_SERVICE)
+            atomic_bytes(AGENT_EXECUTABLE, source, 0o755)
+            atomic_bytes(UNINSTALL_EXECUTABLE, source, 0o755)
+            self.save_journal(journal, phase="verifying", switchedAt=time.time())
+        # The candidate takes the same lock during startup credential recovery.
+        # Never wait for its start or authenticated heartbeat while holding it.
+        systemctl("start", AGENT_SERVICE)
+        if not self.wait_healthy(journal):
+            raise UpdateError("The new agent did not report a healthy authenticated heartbeat; the previous release will be restored")
+        self.check_uninstall()
+        self.load_installation()
+        if any(sha256(read_bytes(path, MAX_SOURCE)) != journal["targetSha256"] for path in (AGENT_EXECUTABLE, UNINSTALL_EXECUTABLE)):
+            raise UpdateError("Installed agent code changed before update verification finished")
+        self.status("updated", current=journal["targetVersion"])
+        self.current_version = journal["targetVersion"]
+        self.finish_journal()
+        return "updated"
+
+    def wait_healthy(self, journal):
+        deadline = time.monotonic() + HEALTH_TIMEOUT
+        while time.monotonic() < deadline:
+            self.check_uninstall()
+            try:
+                active, pid = service_state()
+                health = read_json(AGENT_HEALTH, 4096)
+                checked = health.get("checkedAt")
+                if active and pid > 0 and type(health.get("pid")) is int and health["pid"] == pid and health.get("version") == journal["targetVersion"] and health.get("updateId") == journal["id"] and type(checked) in (float, int) and math.isfinite(checked) and journal["switchedAt"] <= checked <= time.time() + 5:
+                    return True
+            except (UpdateError, OSError, ValueError):
+                pass
+            time.sleep(2)
+        return False
+
+    def rollback(self, journal, error):
+        self.target = {"version": journal["targetVersion"], "sha256": journal["targetSha256"]}
+        self.check_uninstall()
+        self.load_installation()
+        if journal["installation"] != self.installation:
+            raise UpdateError("Agent enrollment changed; rollback needs operator review")
+        backups = [(AGENT_EXECUTABLE, "agent.backup.py", "agentBackupSha256"), (UNINSTALL_EXECUTABLE, "uninstall.backup.py", "uninstallBackupSha256")]
+        sources = []
+        for destination, name, checksum in backups:
+            source = read_bytes(UPDATE_DIR / name, MAX_SOURCE)
+            if sha256(source) != journal[checksum]:
+                raise UpdateError("Agent rollback backup failed integrity verification; repair the installation")
+            source_version(source)
+            sources.append((destination, source))
+        if source_version(sources[0][1]) != journal["previousVersion"]:
+            raise UpdateError("Agent rollback version does not match its recovery journal")
+        self.save_journal(journal, phase="rollback")
+        with self.activity_lock() as activity:
+            if activity is None:
+                raise Deferred("Agent rollback is waiting for the active operation to finish")
+            self.check_uninstall()
+            self.load_installation()
+            systemctl("stop", AGENT_SERVICE)
+            for destination, source in sources:
+                atomic_bytes(destination, source, 0o755)
+            self.save_journal(journal, rollbackSwitchedAt=time.time())
+        systemctl("start", AGENT_SERVICE)
+        rollback_health = {**journal, "targetVersion": journal["previousVersion"], "switchedAt": journal["rollbackSwitchedAt"]}
+        if not self.wait_healthy(rollback_health):
+            raise UpdateError("The previous agent release was restored but its authenticated health is unconfirmed; recovery will retry")
+        self.hold(self.target, error)
+        self.status("failed", error, current=journal["previousVersion"])
+        self.current_version = journal["previousVersion"]
+        self.finish_journal()
+        return "failed"
+
+    def run(self):
+        with exclusive_lock(UPDATE_LOCK) as updater:
+            if updater is None:
+                return "deferred"  # Preserve the active worker's status.
+            recovering = False
+            try:
+                self.check_uninstall()
+                self.load_installation()
+                if UPDATE_JOURNAL.exists() or UPDATE_JOURNAL.is_symlink():
+                    journal = self.read_journal()
+                    self.current_version = journal["previousVersion"]
+                    recovering = True
+                    return self.rollback(journal, "An interrupted agent update was rolled back to the previous release")
+                self.current_version = source_version(read_bytes(AGENT_EXECUTABLE, MAX_SOURCE))
+                self.status("checking")
+                self.target = self.manifest()
+                if version(self.target["version"]) <= version(self.current_version):
+                    return self.status("current")
+                if self.release_held():
+                    return self.status("failed", "This release previously failed verification and is held until the controller publishes a different release")
+                source = self.fetch("/api/agent/release/ec-agent.py", MAX_SOURCE)
+                if sha256(source) != self.target["sha256"]:
+                    raise IntegrityError("The agent release checksum does not match its manifest")
+                if source_version(source) != self.target["version"]:
+                    raise IntegrityError("The agent source VERSION does not match its release manifest")
+                return self.install(source)
+            except Deferred as error:
+                return self.status("deferred", error)
+            except ContactError as error:
+                return self.status("deferred", error)
+            except (UpdateError, OSError, ValueError, KeyError) as error:
+                message = str(error) if isinstance(error, UpdateError) else "Agent update could not finish; inspect the updater service"
+                if UPDATE_JOURNAL.exists():
+                    if recovering:
+                        return self.status("failed", "Agent update recovery is pending; repair the updater service before accepting new operations")
+                    try:
+                        return self.rollback(self.read_journal(), message)
+                    except Deferred as deferred:
+                        return self.status("deferred", deferred)
+                    except (UpdateError, OSError, ValueError, KeyError):
+                        return self.status("failed", "Agent update recovery is pending; repair the updater service before accepting new operations")
+                if isinstance(error, IntegrityError) and self.target:
+                    self.hold(self.target, message)
+                return self.status("failed", message)
+
+
+def main():
+    if os.geteuid() != 0:
+        raise UpdateError("Run the agent updater through its installed root service")
+    Updater().run()
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (UpdateError, OSError, ValueError, KeyError):
+        print("Electric Cherry agent updater failed; inspect its installation and update status", file=sys.stderr)
+        sys.exit(1)

agent/install.sh

@@ -8,6 +8,8 @@ controller=''
 enrollment_token=''
 data_dir='/var/lib/electric-cherry'
 reenroll=''
+existing_enrollment=false
+existing_data_root=false
 while [ "$#" -gt 0 ]; do
     case "$1" in
         --controller) controller="${2:?Missing controller URL}"; shift 2 ;;
@@ -31,7 +33,16 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; }
 
 export DEBIAN_FRONTEND=noninteractive
 apt-get update -qq
-apt-get install -y --no-install-recommends ca-certificates curl python3
+apt-get install -y --no-install-recommends ca-certificates curl python3 util-linux
+installation_preflight() {
+    if [ "$existing_enrollment" = true ] && [ ! -f /etc/electric-cherry/agent.json ]; then
+        echo 'The existing enrollment disappeared during installation. Stop and inspect host deletion before trying again.' >&2
+        return 1
+    fi
+    if [ "$existing_data_root" = true ] && [ ! -d "$data_dir" ]; then
+        echo 'The existing data directory disappeared during installation. Stop and inspect host deletion before trying again.' >&2
+        return 1
+    fi
 python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY'
 # INSTALLATION_PREFLIGHT: this runs before touching managed directories or services.
 import json, pathlib, re, stat, sys
@@ -51,9 +62,14 @@ if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root)
     raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.')
 installed_paths = [config,
     pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-agent.py.update'),
     pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py.update'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-update.py'),
     pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'),
-    pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service')]
+    pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-update.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-update.timer')]
 if any(root == path or root in path.parents for path in installed_paths):
     raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.')
 
@@ -66,10 +82,13 @@ def trusted_directories(path):
         if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
             raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.')
 
-def trusted_json(path):
+def trusted_file(path):
     info = path.lstat()
     if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
         raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.')
+
+def trusted_json(path):
+    trusted_file(path)
     value = json.loads(path.read_text())
     if not isinstance(value, dict):
         raise SystemExit('Existing installation metadata is invalid.')
@@ -89,6 +108,13 @@ if mountinfo.exists():
 request = config.with_name('decommission.json')
 if request.exists() or request.is_symlink():
     raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.')
+trusted_directories(config.parent / 'update')
+journal = config.parent / 'update' / 'journal.json'
+if journal.exists() or journal.is_symlink():
+    raise SystemExit('An automatic update needs recovery. Let electric-cherry-update.service finish or repair that update before rerunning the installer.')
+update_lock = config.with_name('update.lock')
+if update_lock.exists() or update_lock.is_symlink():
+    trusted_file(update_lock)
 saved = None
 if config.exists() or config.is_symlink():
     saved = trusted_json(config)
@@ -108,6 +134,62 @@ elif root.exists() and any(root.iterdir()):
         raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.')
 for directory in ('agent', 'instances', 'libraries'):
     trusted_directories(root / directory)
+activity_lock = root / 'agent' / 'activity.lock'
+if activity_lock.exists() or activity_lock.is_symlink():
+    trusted_file(activity_lock)
+PY
+}
+if [ -f /etc/electric-cherry/agent.json ]; then existing_enrollment=true; fi
+if [ -d "$data_dir" ]; then existing_data_root=true; fi
+installation_preflight
+temporary_dir=$(mktemp -d)
+updater_timer_was_active=false
+update_lock_held=false
+activity_lock_held=false
+installer_finished=false
+installation_cleanup() {
+    local ec_install_status=$?
+    trap - EXIT
+    if [ "$activity_lock_held" = true ]; then
+        flock --unlock 8 || true
+        exec 8>&-
+    fi
+    if [ "$update_lock_held" = true ]; then
+        flock --unlock 9 || true
+        exec 9>&-
+    fi
+    # Recovery must remain scheduled when an upgrade or its second preflight
+    # fails. Release the worker lock before restarting the prior active timer.
+    if [ "$installer_finished" != true ] && [ "$updater_timer_was_active" = true ]; then
+        systemctl start electric-cherry-update.timer >/dev/null 2>&1 || true
+    fi
+    rm -rf "$temporary_dir" || true
+    exit "$ec_install_status"
+}
+trap installation_cleanup EXIT
+controller="${controller%/}"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent-update.py" -o "$temporary_dir/ec-agent-update.py"
+python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" "$temporary_dir/ec-agent-update.py" <<'PY'
+# INSTALLATION_RELEASE: check both downloads before installing or executing either.
+import ast, hashlib, json, pathlib, re, sys
+manifest = json.loads(pathlib.Path(sys.argv[1]).read_text())
+agent = pathlib.Path(sys.argv[2]).read_bytes()
+updater = pathlib.Path(sys.argv[3]).read_bytes()
+if hashlib.sha256(agent).hexdigest() != manifest.get('sha256'):
+    raise SystemExit('Agent release checksum verification failed.')
+if hashlib.sha256(updater).hexdigest() != manifest.get('updaterSha256'):
+    raise SystemExit('Updater release checksum verification failed.')
+if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1 or manifest.get('updaterProtocolVersion') != 1:
+    raise SystemExit('Unsupported agent or updater release manifest.')
+versions = [node.value.value for node in ast.parse(agent).body
+            if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == 'VERSION' for target in node.targets)
+            and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str)]
+if len(versions) != 1 or versions[0] != manifest.get('version') or not re.fullmatch(r'(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)', versions[0]):
+    raise SystemExit('Agent release version does not match its manifest.')
+compile(agent, 'ec-agent.py', 'exec')
+compile(updater, 'ec-agent-update.py', 'exec')
 PY
 if ! command -v docker >/dev/null; then
     if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then
@@ -118,29 +200,37 @@ if ! command -v docker >/dev/null; then
     systemctl enable --now docker
 fi
 docker info >/dev/null 2>&1 || { echo 'Docker is installed but unavailable. Start or repair it, then rerun.' >&2; exit 1; }
-install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances"
-install -d -m 0700 "$data_dir/agent" /etc/electric-cherry
-available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}')
-[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; }
-temporary_dir=$(mktemp -d)
-trap 'rm -rf "$temporary_dir"' EXIT
-controller="${controller%/}"
-curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json"
-curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py"
-python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" <<'PY'
-import hashlib, json, pathlib, sys
-manifest = json.loads(pathlib.Path(sys.argv[1]).read_text())
-actual = hashlib.sha256(pathlib.Path(sys.argv[2]).read_bytes()).hexdigest()
-if actual != manifest.get('sha256'):
-    raise SystemExit('Agent release checksum verification failed.')
-if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1:
-    raise SystemExit('Unsupported agent release manifest.')
-PY
+install -d -m 0700 /etc/electric-cherry
+# Wait for an already-running updater to finish; never interrupt its rollback.
+# The same lock also prevents a timer firing while files are being installed.
+if systemctl cat electric-cherry-update.timer >/dev/null 2>&1; then
+    if systemctl is-active --quiet electric-cherry-update.timer; then
+        updater_timer_was_active=true
+    fi
+    systemctl stop electric-cherry-update.timer
+fi
+exec 9>/etc/electric-cherry/update.lock
+flock --exclusive 9
+update_lock_held=true
+installation_preflight
+# Serialize with running operations, especially a deletion request that could
+# otherwise start between preflight and stopping the main agent.
+if [ -d "$data_dir/agent" ]; then
+    exec 8>"$data_dir/agent/activity.lock"
+    flock --exclusive 8
+    activity_lock_held=true
+    installation_preflight
+fi
 # Stop only the agent while replacing its own executable/journal owner. Existing
 # game containers keep running under Docker throughout installation and updates.
 if systemctl cat electric-cherry-agent.service >/dev/null 2>&1; then
     systemctl stop electric-cherry-agent.service
 fi
+installation_preflight
+install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances"
+install -d -m 0700 "$data_dir/agent"
+available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}')
+[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; }
 install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py 2>/dev/null || {
     install -d -m 0755 /usr/local/libexec
     install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py
@@ -148,6 +238,7 @@ install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-
 # A separate worker survives stopping/removing the agent service. Its only
 # entry point is a fixed, authenticated deletion request prepared by the agent.
 install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py
+install -m 0755 "$temporary_dir/ec-agent-update.py" /usr/local/libexec/electric-cherry-update.py
 printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token"
 unset enrollment_token
 python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"}
@@ -215,8 +306,41 @@ UMask=0077
 [Install]
 WantedBy=multi-user.target
 EOF
+cat > /etc/systemd/system/electric-cherry-update.service <<'EOF'
+[Unit]
+Description=Update the Electric Cherry Host Agent
+After=network-online.target docker.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-update.py
+TimeoutStartSec=0
+UMask=0077
+EOF
+cat > /etc/systemd/system/electric-cherry-update.timer <<'EOF'
+[Unit]
+Description=Check for Electric Cherry Host Agent Updates
+
+[Timer]
+OnBootSec=2min
+OnUnitInactiveSec=5min
+RandomizedDelaySec=30s
+AccuracySec=1s
+Unit=electric-cherry-update.service
+
+[Install]
+WantedBy=timers.target
+EOF
 systemctl daemon-reload
 systemctl enable electric-cherry-agent
+if [ "$activity_lock_held" = true ]; then
+    flock --unlock 8
+    exec 8>&-
+    activity_lock_held=false
+fi
 systemctl restart electric-cherry-agent
 systemctl is-active --quiet electric-cherry-agent || { echo 'Agent service did not start. Inspect journalctl -u electric-cherry-agent.' >&2; exit 1; }
-echo 'Electric Cherry host agent installed and enrolled. Host readiness is shown in Hosts.'
+systemctl enable --now electric-cherry-update.timer
+installer_finished=true
+echo 'Electric Cherry host agent installed and enrolled. Automatic agent updates are enabled; host readiness is shown in Hosts.'

agent/package_release.py

@@ -1,13 +1,37 @@
 #!/usr/bin/env python3
 """Publish the reviewed local agent source into the controller static release."""
+import ast
 import hashlib
 import json
 from pathlib import Path
+import re
 import shutil
 
-source = Path(__file__).resolve().parent
-target = source.parent / "static" / "agent"
-target.mkdir(parents=True, exist_ok=True)
-shutil.copyfile(source / "ec_agent.py", target / "ec-agent.py")
-shutil.copyfile(source / "install.sh", target / "install.sh")
-(target / "manifest.json").write_text(json.dumps({"version": "1.1.0", "protocolVersion": 1, "platform": "debian-amd64", "sha256": hashlib.sha256((target / "ec-agent.py").read_bytes()).hexdigest()}, indent=2) + "\n")
+
+def package(source, target):
+    """Read version metadata without importing or executing release code."""
+    source, target = Path(source), Path(target)
+    agent = (source / "ec_agent.py").read_bytes()
+    updater = (source / "ec_agent_update.py").read_bytes()
+    versions = [node.value.value for node in ast.parse(agent).body
+                if isinstance(node, ast.Assign) and any(isinstance(name, ast.Name) and name.id == "VERSION" for name in node.targets)
+                and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str)]
+    if len(versions) != 1 or not re.fullmatch(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", versions[0]):
+        raise ValueError("Agent source must declare exactly one literal semantic VERSION")
+    compile(agent, "ec_agent.py", "exec")
+    compile(updater, "ec_agent_update.py", "exec")
+    manifest = {"version": versions[0], "protocolVersion": 1, "platform": "debian-amd64",
+                "sha256": hashlib.sha256(agent).hexdigest(), "updaterProtocolVersion": 1,
+                "updaterSha256": hashlib.sha256(updater).hexdigest()}
+    target.mkdir(parents=True, exist_ok=True)
+    (target / "ec-agent.py").write_bytes(agent)
+    (target / "ec-agent-update.py").write_bytes(updater)
+    shutil.copyfile(source / "install.sh", target / "install.sh")
+    temporary_manifest = target / ".manifest.json.tmp"
+    temporary_manifest.write_text(json.dumps(manifest, indent=2) + "\n")
+    temporary_manifest.replace(target / "manifest.json")
+
+
+if __name__ == "__main__":
+    source = Path(__file__).resolve().parent
+    package(source, source.parent / "static" / "agent")

agent/tests/test_installer.py

@@ -1,10 +1,13 @@
 """Exercise installer guards without installing software or touching services."""
 
+import hashlib
+import importlib.util
 import json
 import os
 from pathlib import Path
 import re
 import stat
+import subprocess
 import sys
 import tempfile
 import unittest
@@ -15,6 +18,11 @@ SOURCE = (Path(__file__).resolve().parents[1] / "install.sh").read_text()
 BLOCKS = re.findall(r"<<'PY'\n(.*?)\nPY", SOURCE, re.DOTALL)
 PREFLIGHT = next(block for block in BLOCKS if "# INSTALLATION_PREFLIGHT:" in block)
 OWNERSHIP = next(block for block in BLOCKS if "# INSTALLATION_OWNERSHIP:" in block)
+RELEASE = next(block for block in BLOCKS if "# INSTALLATION_RELEASE:" in block)
+CLEANUP = re.search(r"installation_cleanup\(\) \{.*?\n\}", SOURCE, re.DOTALL).group()
+SPEC = importlib.util.spec_from_file_location("package_release", Path(__file__).resolve().parents[1] / "package_release.py")
+PACKAGER = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(PACKAGER)
 
 
 class InstallerTests(unittest.TestCase):
@@ -163,6 +171,47 @@ class InstallerTests(unittest.TestCase):
         with self.assertRaisesRegex(SystemExit, "deletion is in progress"):
             self.preflight()
 
+    def test_pending_update_recovery_is_preserved_and_blocks_manual_replacement(self):
+        self.legacy_installation()
+        update = self.config.parent / "update"
+        update.mkdir()
+        journal = update / "journal.json"
+        saved = '{"phase":"swapping","targetVersion":"1.2.0"}'
+        journal.write_text(saved)
+        with self.assertRaisesRegex(SystemExit, "automatic update needs recovery"):
+            self.preflight()
+        self.assertEqual(journal.read_text(), saved)
+        journal.unlink()
+        journal.symlink_to(self.base / "missing")
+        with self.assertRaisesRegex(SystemExit, "automatic update needs recovery"):
+            self.preflight()
+
+    def test_update_directory_and_lock_cannot_follow_symlinks(self):
+        self.legacy_installation()
+        elsewhere = self.base / "unrelated"
+        elsewhere.mkdir()
+        update = self.config.parent / "update"
+        update.symlink_to(elsewhere, target_is_directory=True)
+        with self.assertRaisesRegex(SystemExit, "not symlinks"):
+            self.preflight()
+        update.unlink()
+        lock = self.config.with_name("update.lock")
+        lock.symlink_to(elsewhere / "sentinel")
+        with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
+            self.preflight()
+        self.assertFalse((elsewhere / "sentinel").exists())
+        lock.unlink()
+        activity = self.root / "agent" / "activity.lock"
+        activity.symlink_to(elsewhere / "activity-sentinel")
+        with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
+            self.preflight()
+        self.assertFalse((elsewhere / "activity-sentinel").exists())
+
+    def test_data_root_cannot_contain_the_updater_or_service_installation(self):
+        for path in ("/etc/systemd", "/etc/systemd/system", "/usr/local/libexec/electric-cherry-update.py", "/usr/local/libexec/electric-cherry-agent.py.update", "/usr/local/libexec/electric-cherry-uninstall.py.update"):
+            with self.subTest(path=path), self.assertRaisesRegex(SystemExit, "outside the agent executable and service"):
+                self.preflight(path)
+
     def test_mounted_parent_is_allowed_but_data_root_and_nested_mounts_are_rejected(self):
         mountinfo = Path('/proc/1/mountinfo')
         original_exists, original_read = Path.exists, Path.read_text
@@ -181,5 +230,125 @@ class InstallerTests(unittest.TestCase):
                         self.preflight()
 
 
+class ReleaseTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.source = Path(self.temporary.name) / "source"
+        self.target = Path(self.temporary.name) / "release"
+        self.source.mkdir()
+        (self.source / "ec_agent.py").write_text('VERSION = "3.12.7"\nraise RuntimeError("must never execute during packaging")\n')
+        (self.source / "ec_agent_update.py").write_text('raise RuntimeError("must never execute during verification")\n')
+        (self.source / "install.sh").write_text("#!/bin/bash\n")
+        PACKAGER.package(self.source, self.target)
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def verify(self):
+        argv = ["-", str(self.target / "manifest.json"), str(self.target / "ec-agent.py"), str(self.target / "ec-agent-update.py")]
+        with patch.object(sys, "argv", argv):
+            exec(compile(RELEASE, "installer-release", "exec"), {})
+
+    def manifest(self, **changes):
+        path = self.target / "manifest.json"
+        value = json.loads(path.read_text())
+        value.update(changes)
+        path.write_text(json.dumps(value))
+        return value
+
+    def test_package_derives_version_and_hashes_both_sources_without_executing_them(self):
+        manifest = self.manifest()
+        self.assertEqual(manifest["version"], "3.12.7")
+        self.assertEqual(manifest["updaterProtocolVersion"], 1)
+        self.assertEqual(manifest["sha256"], hashlib.sha256((self.source / "ec_agent.py").read_bytes()).hexdigest())
+        self.assertEqual(manifest["updaterSha256"], hashlib.sha256((self.source / "ec_agent_update.py").read_bytes()).hexdigest())
+        self.assertEqual((self.target / "ec-agent-update.py").read_bytes(), (self.source / "ec_agent_update.py").read_bytes())
+        self.verify()
+
+    def test_changed_agent_and_updater_downloads_are_rejected(self):
+        for filename, error in (("ec-agent.py", "Agent release checksum"), ("ec-agent-update.py", "Updater release checksum")):
+            with self.subTest(filename=filename):
+                path = self.target / filename
+                original = path.read_bytes()
+                path.write_bytes(original + b"# changed\n")
+                with self.assertRaisesRegex(SystemExit, error):
+                    self.verify()
+                path.write_bytes(original)
+
+    def test_installer_rejects_missing_or_unsupported_updater_protocol(self):
+        for protocol in (None, 0, 2, "1"):
+            self.manifest(updaterProtocolVersion=protocol)
+            with self.subTest(protocol=protocol), self.assertRaisesRegex(SystemExit, "Unsupported agent or updater"):
+                self.verify()
+
+    def test_installer_rejects_manifest_version_mismatch_and_invalid_code(self):
+        self.manifest(version="3.12.8")
+        with self.assertRaisesRegex(SystemExit, "version does not match"):
+            self.verify()
+        malformed = b"def malformed(:\n"
+        (self.target / "ec-agent-update.py").write_bytes(malformed)
+        self.manifest(version="3.12.7", updaterSha256=hashlib.sha256(malformed).hexdigest())
+        with self.assertRaises(SyntaxError):
+            self.verify()
+
+    def test_packaging_requires_one_literal_semantic_agent_version(self):
+        for source in ('VERSION = "bad"', 'VERSION = "01.2.3"', 'VERSION = str(3)', 'VERSION = "1.2.3"\nVERSION = "1.2.4"'):
+            (self.source / "ec_agent.py").write_text(source)
+            with self.subTest(source=source), self.assertRaisesRegex(ValueError, "literal semantic VERSION"):
+                PACKAGER.package(self.source, self.target)
+
+    def test_installer_serializes_changes_and_enables_update_timer_only_after_agent_starts(self):
+        locked = SOURCE.index("flock --exclusive 9")
+        self.assertLess(SOURCE.index("systemctl stop electric-cherry-update.timer"), locked)
+        self.assertLess(locked, SOURCE.index("installation_preflight", locked))
+        self.assertLess(SOURCE.index("installation_preflight", locked), SOURCE.index('install -m 0755 "$temporary_dir/ec-agent.py"'))
+        self.assertLess(locked, SOURCE.index("flock --exclusive 8"))
+        self.assertLess(SOURCE.index("flock --exclusive 8"), SOURCE.index("systemctl stop electric-cherry-agent.service"))
+        self.assertLess(SOURCE.index("flock --unlock 8", SOURCE.index("systemctl daemon-reload")), SOURCE.index("systemctl restart electric-cherry-agent"))
+        self.assertLess(SOURCE.index("systemctl is-active --quiet electric-cherry-agent"), SOURCE.index("systemctl enable --now electric-cherry-update.timer"))
+        self.assertIn("ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-update.py", SOURCE)
+        self.assertIn("OnUnitInactiveSec=5min", SOURCE)
+        self.assertIn("RandomizedDelaySec=30s", SOURCE)
+        self.assertIn("AccuracySec=1s", SOURCE)
+        self.assertNotIn("Persistent=true", SOURCE)
+
+    def test_failed_install_restores_only_previously_active_timer_after_releasing_locks(self):
+        for previous, finished, code in ((True, False, 37), (False, False, 37), (True, True, 0)):
+            with self.subTest(previous=previous, finished=finished):
+                scratch = Path(self.temporary.name) / "scratch"
+                scratch.mkdir(exist_ok=True)
+                log = Path(self.temporary.name) / "cleanup.log"
+                log.unlink(missing_ok=True)
+                script = CLEANUP + r'''
+flock() { printf 'unlock:%s\n' "$2" >> "$EC_TEST_LOG"; }
+systemctl() {
+    if { : >&8; } 2>/dev/null || { : >&9; } 2>/dev/null; then
+        printf 'lock-still-open\n' >> "$EC_TEST_LOG"
+        return 1
+    fi
+    printf '%s\n' "$*" >> "$EC_TEST_LOG"
+}
+temporary_dir="$EC_TEST_TEMP"
+updater_timer_was_active="$EC_TEST_PREVIOUS"
+installer_finished="$EC_TEST_FINISHED"
+update_lock_held=true
+activity_lock_held=true
+exec 8>"$EC_TEST_LOCK"
+exec 9>"$EC_TEST_LOCK"
+trap installation_cleanup EXIT
+exit "$EC_TEST_EXIT"
+'''
+                environment = {**os.environ, "EC_TEST_LOG": str(log), "EC_TEST_TEMP": str(scratch),
+                               "EC_TEST_PREVIOUS": str(previous).lower(), "EC_TEST_FINISHED": str(finished).lower(),
+                               "EC_TEST_LOCK": str(Path(self.temporary.name) / "test.lock"), "EC_TEST_EXIT": str(code)}
+                result = subprocess.run(["bash", "-c", script], env=environment, capture_output=True)
+                self.assertEqual(result.returncode, code, result.stderr)
+                expected = ["unlock:8", "unlock:9"]
+                if previous and not finished:
+                    expected.append("start electric-cherry-update.timer")
+                self.assertEqual(log.read_text().splitlines(), expected)
+                self.assertFalse(scratch.exists())
+
+
 if __name__ == "__main__":
     unittest.main()

agent/tests/test_uninstall.py

@@ -57,6 +57,9 @@ class UninstallTests(unittest.TestCase):
             "UNINSTALL_EXECUTABLE": self.bin_dir / "electric-cherry-uninstall.py",
             "UNINSTALL_UNIT": self.unit_dir / "electric-cherry-uninstall.service",
             "UNINSTALL_REQUEST": self.config_dir / "decommission.json",
+            "UPDATE_EXECUTABLE": self.bin_dir / "electric-cherry-update.py",
+            "UPDATE_UNIT": self.unit_dir / "electric-cherry-update.service",
+            "UPDATE_TIMER": self.unit_dir / "electric-cherry-update.timer",
         }
         self.patches = [patch.object(module, key, value) for key, value in replacements.items()]
         # Unit tests run without root privileges. Keep actual path, symlink,
@@ -74,8 +77,10 @@ class UninstallTests(unittest.TestCase):
         self.config = {"hostId": "host-one", "controller": "https://controller.example", "credential": "secret", "root": str(self.root), "configPath": str(module.AGENT_CONFIG)}
         self.agent = module.Agent(self.config)
         self.docker = CleanupDocker()
-        for path in (module.AGENT_EXECUTABLE, module.AGENT_UNIT, module.UNINSTALL_EXECUTABLE, module.UNINSTALL_UNIT):
+        for path in (module.AGENT_EXECUTABLE, module.AGENT_UNIT, module.UNINSTALL_EXECUTABLE, module.UNINSTALL_UNIT, module.UPDATE_EXECUTABLE, module.UPDATE_UNIT, module.UPDATE_TIMER):
             path.write_text("installed")
+        for path in (module.AGENT_EXECUTABLE, module.UNINSTALL_EXECUTABLE):
+            path.with_suffix(".py.update").write_text("interrupted update")
         module.atomic_json(module.AGENT_CONFIG, self.config)
         module.atomic_json(self.root / module.ROOT_MARKER, {"version": 1, "root": str(self.root), "hostId": "host-one", "controller": "https://controller.example"})
         self.game = self.root / "libraries" / "t6" / "library" / "content" / "zone" / "all" / "dlc.ff"
@@ -108,6 +113,11 @@ class UninstallTests(unittest.TestCase):
             self.assertFalse(module.AGENT_CONFIG.exists())
             self.assertFalse(module.AGENT_EXECUTABLE.exists())
             self.assertFalse(module.AGENT_UNIT.exists())
+            self.assertFalse(module.UPDATE_EXECUTABLE.exists())
+            self.assertFalse(module.UPDATE_UNIT.exists())
+            self.assertFalse(module.UPDATE_TIMER.exists())
+            self.assertFalse(module.AGENT_EXECUTABLE.with_suffix(".py.update").exists())
+            self.assertFalse(module.UNINSTALL_EXECUTABLE.with_suffix(".py.update").exists())
             if stage == "cleaned":
                 self.assertTrue(module.UNINSTALL_REQUEST.exists())
                 self.assertTrue(module.UNINSTALL_EXECUTABLE.exists())
@@ -120,6 +130,8 @@ class UninstallTests(unittest.TestCase):
         self.assertEqual(stages, ["cleaned", "completed"])
         self.assertEqual((self.base / "unrelated.txt").read_text(), "keep")
         module.systemctl.assert_any_call("disable", "--now", "electric-cherry-agent.service")
+        module.systemctl.assert_any_call("disable", "--now", "electric-cherry-update.timer")
+        module.systemctl.assert_any_call("stop", "electric-cherry-update.service")
         module.systemctl.assert_any_call("disable", "electric-cherry-uninstall.service")
 
     def test_handoff_is_durable_and_never_reports_operation_completed(self):

agent/tests/test_update_integration.py

@@ -0,0 +1,137 @@
+import fcntl
+import importlib.util
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("ec_update_integration", Path(__file__).resolve().parents[1] / "ec_agent.py")
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+
+class UpdateIntegrationTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.base = Path(self.temporary.name)
+        self.config_dir = self.base / "config"
+        self.config_dir.mkdir()
+        replacements = {
+            "AGENT_CONFIG": self.config_dir / "agent.json",
+            "UPDATE_JOURNAL": self.config_dir / "update" / "journal.json",
+            "UPDATE_STATUS": self.config_dir / "update-status.json",
+            "AGENT_HEALTH": self.config_dir / "agent-health.json",
+        }
+        self.patches = [patch.object(module, key, value) for key, value in replacements.items()]
+        self.patches.append(patch.object(module, "protected_file", side_effect=lambda path, **_: path))
+        for entry in self.patches:
+            entry.start()
+        self.agent = module.Agent({"root": str(self.base / "data"), "configPath": str(module.AGENT_CONFIG), "controller": "https://controller.example", "hostId": "one", "credential": "secret"})
+        self.agent.can_update = lambda: True
+        self.agent.inventory = lambda: {"agentVersion": module.VERSION}
+
+    def tearDown(self):
+        self.agent.journal.close()
+        for entry in reversed(self.patches):
+            entry.stop()
+        self.temporary.cleanup()
+
+    def assert_activity_held(self):
+        with (self.agent.root / "agent" / "activity.lock").open("a") as stream:
+            with self.assertRaises(BlockingIOError):
+                fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
+
+    def test_operation_delivery_execution_and_reports_exclude_updates(self):
+        operation = {"id": "one", "kind": "host.inspect", "payload": {}}
+        calls = []
+        def request(method, path, body=None):
+            self.assert_activity_held()
+            calls.append((method, path))
+            return {"operations": [operation]} if method == "GET" else {}
+        def dispatch(value):
+            self.assert_activity_held()
+            return {"ok": True}
+        self.agent.request = request
+        self.agent.dispatch = dispatch
+        self.agent.poll_operations()
+        self.assertEqual(self.agent.journal.get("one")["state"], "completed")
+        self.assertEqual(len(calls), 4)
+        self.assertIsNone(self.agent.current_operation)
+        with self.agent.activity() as acquired:
+            self.assertTrue(acquired)
+
+    def test_updater_lock_defers_delivery_without_blocking_health(self):
+        self.agent.request = lambda method, path, body=None: {"host": {}}
+        with self.agent.activity() as acquired:
+            self.assertTrue(acquired)
+            with patch.object(self.agent, "request", wraps=self.agent.request) as request:
+                self.agent.poll_operations()
+                request.assert_not_called()
+                self.agent.heartbeat()
+                self.assertTrue(module.AGENT_HEALTH.exists())
+
+    def test_journal_blocks_operations_during_verification_and_recovery(self):
+        module.UPDATE_JOURNAL.parent.mkdir()
+        for content in ('{"id":"candidate","phase":"verifying"}', '{"id":"candidate","phase":"rollback"}', '{broken'):
+            module.UPDATE_JOURNAL.write_text(content)
+            with patch.object(self.agent, "request") as request:
+                self.agent.poll_operations()
+                request.assert_not_called()
+        module.UPDATE_JOURNAL.unlink()
+        with patch.object(self.agent, "request", return_value={"operations": []}) as request:
+            self.agent.poll_operations()
+            request.assert_called_once_with("GET", "/api/agent/operations")
+
+    def test_failed_controller_contact_never_writes_healthy_receipt(self):
+        with patch.object(self.agent, "request", side_effect=module.AgentError("offline")):
+            with self.assertRaises(module.AgentError):
+                self.agent.heartbeat()
+        self.assertFalse(module.AGENT_HEALTH.exists())
+
+    def test_health_proves_current_process_version_and_transaction(self):
+        module.atomic_json(module.UPDATE_JOURNAL, {"id": "candidate-id", "phase": "verifying"})
+        with patch.object(self.agent, "request", return_value={"host": {}}), patch.object(module.time, "time", return_value=12345.5):
+            self.agent.heartbeat()
+        self.assertEqual(json.loads(module.AGENT_HEALTH.read_text()), {"version": module.VERSION, "pid": os.getpid(), "checkedAt": 12345.5, "updateId": "candidate-id"})
+        module.UPDATE_JOURNAL.unlink()
+        self.agent.record_health()
+        self.assertNotIn("updateId", json.loads(module.AGENT_HEALTH.read_text()))
+
+    def test_real_agent_health_receipt_is_accepted_by_the_independent_updater(self):
+        updater_spec = importlib.util.spec_from_file_location("ec_update_contract", Path(__file__).resolve().parents[1] / "ec_agent_update.py")
+        updater_module = importlib.util.module_from_spec(updater_spec)
+        updater_spec.loader.exec_module(updater_module)
+        journal = {"id": "candidate-id", "phase": "verifying", "targetVersion": module.VERSION, "switchedAt": module.time.time() - 1}
+        module.atomic_json(module.UPDATE_JOURNAL, journal)
+        with patch.object(self.agent, "request", return_value={"host": {}}):
+            self.agent.heartbeat()
+        updater = updater_module.Updater()
+        with patch.object(updater, "check_uninstall"), patch.object(updater_module, "AGENT_HEALTH", module.AGENT_HEALTH), patch.object(updater_module, "trusted", side_effect=lambda path, **_: path), patch.object(updater_module, "service_state", return_value=(True, os.getpid())):
+            self.assertTrue(updater.wait_healthy(journal))
+
+    def test_operation_exception_releases_update_lock(self):
+        with patch.object(self.agent, "request", side_effect=module.AgentError("offline")):
+            with self.assertRaises(module.AgentError):
+                self.agent.poll_operations()
+        with self.agent.activity() as acquired:
+            self.assertTrue(acquired)
+
+    def test_update_inventory_excludes_local_recovery_and_secret_fields(self):
+        module.atomic_json(module.UPDATE_STATUS, {"state": "failed", "targetVersion": "1.3.0", "currentVersion": "untrusted", "error": "Restored previous agent", "checkedAt": 1700000000, "credential": "secret", "journal": {"private": True}, "failedSha256": "digest"})
+        self.assertEqual(self.agent.update_inventory(), {"enabled": True, "state": "failed", "currentVersion": module.VERSION, "targetVersion": "1.3.0", "error": "Restored previous agent", "checkedAt": "2023-11-14T22:13:20Z"})
+
+    def test_malformed_update_status_is_visible_without_breaking_inventory(self):
+        module.UPDATE_STATUS.write_text("[]")
+        self.assertEqual(self.agent.update_inventory()["state"], "failed")
+
+    def test_shutdown_never_polls_work(self):
+        self.agent.stopping.set()
+        with patch.object(self.agent, "request") as request:
+            self.agent.poll_operations()
+            request.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()

agent/tests/test_updates.py

@@ -0,0 +1,418 @@
+import contextlib
+import fcntl
+import importlib.util
+import io
+import json
+import os
+from pathlib import Path
+import stat
+import tempfile
+import time
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("ec_agent_update", Path(__file__).resolve().parents[1] / "ec_agent_update.py")
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+PREVIOUS = b'VERSION = "1.0.0"\n# previous agent\n'
+CANDIDATE = b'VERSION = "1.1.0"\n# next agent\n'
+
+
+class PowerLoss(BaseException):
+    pass
+
+
+class Response(io.BytesIO):
+    def __init__(self, value, headers=None):
+        super().__init__(value)
+        self.headers = headers or {}
+        self.status = 200
+
+
+class UpdateTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.base = Path(self.temp.name).resolve()
+        self.config_dir = self.base / "etc" / "electric-cherry"
+        self.bin_dir = self.base / "libexec"
+        self.root = self.base / "data"
+        for directory in (self.config_dir, self.bin_dir, self.root / "agent"):
+            directory.mkdir(parents=True)
+        paths = {
+            "AGENT_CONFIG": self.config_dir / "agent.json",
+            "AGENT_EXECUTABLE": self.bin_dir / "electric-cherry-agent.py",
+            "UNINSTALL_EXECUTABLE": self.bin_dir / "electric-cherry-uninstall.py",
+            "UNINSTALL_REQUEST": self.config_dir / "decommission.json",
+            "UPDATE_LOCK": self.config_dir / "update.lock",
+            "UPDATE_DIR": self.config_dir / "update",
+            "UPDATE_JOURNAL": self.config_dir / "update" / "journal.json",
+            "UPDATE_STATUS": self.config_dir / "update-status.json",
+            "AGENT_HEALTH": self.config_dir / "agent-health.json",
+        }
+        self.patches = [patch.object(module, name, value) for name, value in paths.items()]
+        def trusted_for_test(value, missing=False, directory=False):
+            path = Path(value)
+            if any(item.is_symlink() for item in (path, *path.parents)):
+                raise module.UpdateError("Unsafe symbolic link")
+            if not path.exists():
+                if missing:
+                    return path
+                raise module.UpdateError("File unavailable")
+            if directory and not path.is_dir() or not directory and not path.is_file():
+                raise module.UpdateError("Wrong file type")
+            return path
+        self.patches.extend([patch.object(module, "trusted", side_effect=trusted_for_test), patch.object(module, "sync_parent"), patch.object(module, "systemctl", side_effect=self.systemctl)])
+        for item in self.patches:
+            item.start()
+        self.config = {"hostId": "host-one", "root": str(self.root), "controller": "https://controller.example", "credential": "secret-enrollment-credential"}
+        self.config_bytes = json.dumps(self.config).encode()
+        module.AGENT_CONFIG.write_bytes(self.config_bytes)
+        (self.root / ".electric-cherry.json").write_text(json.dumps({"version": 1, "root": str(self.root), "hostId": "host-one", "controller": "https://controller.example"}))
+        module.AGENT_EXECUTABLE.write_bytes(PREVIOUS)
+        module.UNINSTALL_EXECUTABLE.write_bytes(PREVIOUS)
+        (self.root / "games.ff").write_bytes(b"large game files unchanged")
+        self.commands = []
+        self.active = True
+        self.pid = 123
+        self.write_health = True
+        self.downloads = []
+        self.source = CANDIDATE
+        self.release = {"version": "1.1.0", "sha256": module.sha256(CANDIDATE), "protocolVersion": 1, "updaterProtocolVersion": 1, "platform": "debian-amd64"}
+
+    def tearDown(self):
+        for item in reversed(self.patches):
+            item.stop()
+        self.temp.cleanup()
+
+    def systemctl(self, *args):
+        self.commands.append(args)
+        if args == ("stop", module.AGENT_SERVICE):
+            self.active = False
+            return ""
+        if args == ("start", module.AGENT_SERVICE):
+            # The candidate must be able to take the operation lock at startup.
+            with module.exclusive_lock(self.root / "agent" / "activity.lock") as activity:
+                self.assertIsNotNone(activity, "Updater held the operation lock while starting the agent")
+            self.active = True
+            self.pid += 1
+            if self.write_health and module.UPDATE_JOURNAL.exists():
+                journal = json.loads(module.UPDATE_JOURNAL.read_text())
+                module.AGENT_HEALTH.write_text(json.dumps({"version": module.source_version(module.AGENT_EXECUTABLE.read_bytes()), "pid": self.pid, "checkedAt": time.time(), "updateId": journal["id"]}))
+            return ""
+        if args == ("show", module.AGENT_SERVICE, "--property=ActiveState", "--property=MainPID"):
+            return "ActiveState=" + ("active" if self.active else "inactive") + "\nMainPID=" + str(self.pid if self.active else 0) + "\n"
+        self.fail("Unexpected service command " + str(args))
+
+    def fetch(self, path, limit):
+        self.downloads.append(path)
+        if path.endswith("manifest.json"):
+            return json.dumps(self.release).encode()
+        if path.endswith("ec-agent.py"):
+            return self.source
+        self.fail("Unexpected download " + path)
+
+    def updater(self):
+        updater = module.Updater()
+        updater.fetch = self.fetch
+        return updater
+
+    def status(self):
+        return json.loads(module.UPDATE_STATUS.read_text())
+
+    def assert_unchanged(self):
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), PREVIOUS)
+        self.assertEqual(module.UNINSTALL_EXECUTABLE.read_bytes(), PREVIOUS)
+        self.assertEqual(module.AGENT_CONFIG.read_bytes(), self.config_bytes)
+        self.assertEqual((self.root / "games.ff").read_bytes(), b"large game files unchanged")
+
+    def test_verified_upgrade_waits_for_authenticated_candidate_health(self):
+        self.assertEqual(self.updater().run(), "updated")
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(module.UNINSTALL_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(module.AGENT_CONFIG.read_bytes(), self.config_bytes)
+        self.assertEqual((self.root / "games.ff").read_bytes(), b"large game files unchanged")
+        self.assertEqual(self.status()["currentVersion"], "1.1.0")
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+        self.assertFalse((module.UPDATE_DIR / "agent.backup.py").exists())
+        self.assertEqual(next(command for command in self.commands if command[0] != "show"), ("stop", module.AGENT_SERVICE))
+
+    def test_equal_or_older_versions_do_not_download_or_restart(self):
+        for target in ("1.0.0", "0.9.9"):
+            with self.subTest(target=target):
+                self.release["version"] = target
+                self.assertEqual(self.updater().run(), "current")
+                self.assert_unchanged()
+        self.assertFalse(any(path.endswith("ec-agent.py") for path in self.downloads))
+        self.assertEqual(self.commands, [])
+
+    def test_busy_operation_defers_update_without_stopping_agent(self):
+        with module.exclusive_lock(self.root / "agent" / "activity.lock"):
+            self.assertEqual(self.updater().run(), "deferred")
+        self.assertIn("executing an operation", self.status()["error"])
+        self.assert_unchanged()
+        self.assertEqual(self.commands, [])
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+
+    def test_deliberately_stopped_agent_is_not_updated_or_started(self):
+        self.active = False
+        self.assertEqual(self.updater().run(), "deferred")
+        self.assertIn("Agent service is not running", self.status()["error"])
+        self.assert_unchanged()
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+        self.assertFalse(self.active)
+        self.assertFalse(any(command[0] in ("start", "stop") for command in self.commands))
+
+    def test_concurrent_worker_does_not_overwrite_active_worker_status(self):
+        module.UPDATE_STATUS.write_text('{"state":"updating"}')
+        with module.exclusive_lock(module.UPDATE_LOCK):
+            self.assertEqual(self.updater().run(), "deferred")
+        self.assertEqual(self.status(), {"state": "updating"})
+        self.assertEqual(self.downloads, [])
+
+    def test_uninstall_request_prevents_any_download_or_mutation(self):
+        module.UNINSTALL_REQUEST.write_text("{}")
+        self.assertEqual(self.updater().run(), "deferred")
+        self.assertIn("deletion", self.status()["error"])
+        self.assertEqual(self.downloads, [])
+        self.assertEqual(self.commands, [])
+        self.assert_unchanged()
+
+    def test_uninstall_starting_during_download_wins_before_code_changes(self):
+        updater = self.updater()
+        def fetch(path, limit):
+            data = self.fetch(path, limit)
+            if path.endswith("ec-agent.py"):
+                module.UNINSTALL_REQUEST.write_text("{}")
+            return data
+        updater.fetch = fetch
+        self.assertEqual(updater.run(), "deferred")
+        self.assert_unchanged()
+        self.assertEqual(self.commands, [])
+
+    def test_controller_unavailable_keeps_running_agent(self):
+        updater = self.updater()
+        updater.fetch = lambda *_: (_ for _ in ()).throw(module.ContactError("Controller unavailable"))
+        self.assertEqual(updater.run(), "deferred")
+        self.assert_unchanged()
+        self.assertTrue(self.active)
+        self.assertEqual(self.commands, [])
+
+    def test_checksum_failure_holds_artifact_without_executing_it(self):
+        self.source = CANDIDATE + b"# changed in transit"
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertIn("checksum", self.status()["error"])
+        self.assert_unchanged()
+        self.assertEqual(self.commands, [])
+        self.downloads.clear()
+        self.source = CANDIDATE
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertEqual(self.downloads, ["/api/agent/release/manifest.json"])
+
+    def test_invalid_or_mismatched_python_version_is_never_installed(self):
+        for source in (b'VERSION = "1.1.0"\ninvalid syntax ?\n', b'VERSION = "5.0.0"\n'):
+            with self.subTest(source=source):
+                self.source = source
+                self.release["sha256"] = module.sha256(source)
+                self.assertEqual(self.updater().run(), "failed")
+                self.assert_unchanged()
+                self.assertEqual(self.commands, [])
+
+    def test_incompatible_protocol_platform_or_version_never_downloads_code(self):
+        original = dict(self.release)
+        for key, value in (("protocolVersion", 2), ("protocolVersion", True), ("platform", "linux-arm64"), ("version", "1.1.0-rc1"), ("version", "01.1.0"), ("updaterProtocolVersion", 2)):
+            with self.subTest(key=key, value=value):
+                self.release = {**original, key: value}
+                self.assertIn(self.updater().run(), ("failed", "deferred"))
+                self.assert_unchanged()
+        self.assertFalse(any(path.endswith("ec-agent.py") for path in self.downloads))
+        self.assertEqual(self.commands, [])
+
+    def test_failed_health_rolls_back_both_files_then_holds_release(self):
+        updater = self.updater()
+        updater.wait_healthy = lambda journal: journal["targetVersion"] == "1.0.0"
+        self.assertEqual(updater.run(), "failed")
+        self.assert_unchanged()
+        self.assertTrue(self.active)
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+        self.assertEqual(self.status()["currentVersion"], "1.0.0")
+        self.assertEqual(sum(command == ("start", module.AGENT_SERVICE) for command in self.commands), 2)
+        self.downloads.clear()
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertEqual(self.downloads, ["/api/agent/release/manifest.json"])
+
+    def test_changed_artifact_of_failed_target_can_be_retried(self):
+        updater = self.updater()
+        updater.wait_healthy = lambda journal: journal["targetVersion"] == "1.0.0"
+        self.assertEqual(updater.run(), "failed")
+        self.source = CANDIDATE + b"# corrected release\n"
+        self.release["sha256"] = module.sha256(self.source)
+        self.assertEqual(self.updater().run(), "updated")
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), self.source)
+
+    def crash_during_second_replacement(self):
+        original = module.atomic_bytes
+        def interrupt(path, value, mode=0o600):
+            if Path(path) == module.UNINSTALL_EXECUTABLE and value == CANDIDATE:
+                raise PowerLoss()
+            return original(path, value, mode)
+        with patch.object(module, "atomic_bytes", side_effect=interrupt):
+            with self.assertRaises(PowerLoss):
+                self.updater().run()
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(module.UNINSTALL_EXECUTABLE.read_bytes(), PREVIOUS)
+        self.assertTrue(module.UPDATE_JOURNAL.exists())
+
+    def test_crash_between_executable_replacements_recovers_from_backups(self):
+        self.crash_during_second_replacement()
+        self.downloads.clear()
+        self.assertEqual(self.updater().run(), "failed")
+        self.assert_unchanged()
+        self.assertEqual(self.downloads, [])
+        self.assertTrue(self.active)
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+
+    def test_crash_after_verification_still_recovers_conservatively(self):
+        updater = self.updater()
+        updater.finish_journal = lambda: (_ for _ in ()).throw(PowerLoss())
+        with self.assertRaises(PowerLoss):
+            updater.run()
+        self.assertEqual(json.loads(module.UPDATE_JOURNAL.read_text())["phase"], "verifying")
+        self.assertEqual(self.updater().run(), "failed")
+        self.assert_unchanged()
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+
+    def test_corrupt_backup_keeps_journal_and_never_overwrites_code(self):
+        self.crash_during_second_replacement()
+        (module.UPDATE_DIR / "agent.backup.py").write_bytes(b"corrupt")
+        before = len(self.commands)
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertTrue(module.UPDATE_JOURNAL.exists())
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(len(self.commands), before)
+
+    def test_changed_enrollment_prevents_update_and_rollback(self):
+        self.crash_during_second_replacement()
+        changed = {**self.config, "hostId": "different-host"}
+        module.AGENT_CONFIG.write_text(json.dumps(changed))
+        (self.root / ".electric-cherry.json").write_text(json.dumps({"version": 1, "root": str(self.root), "hostId": "different-host", "controller": "https://controller.example"}))
+        before = len(self.commands)
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertTrue(module.UPDATE_JOURNAL.exists())
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(len(self.commands), before)
+
+    def test_rollback_busy_lock_keeps_recovery_journal_for_next_tick(self):
+        self.crash_during_second_replacement()
+        with module.exclusive_lock(self.root / "agent" / "activity.lock"):
+            self.assertEqual(self.updater().run(), "deferred")
+        self.assertEqual(json.loads(module.UPDATE_JOURNAL.read_text())["phase"], "rollback")
+        self.assertEqual(self.updater().run(), "failed")
+        self.assert_unchanged()
+
+    def test_unconfirmed_rollback_health_retains_backups_and_journal(self):
+        updater = self.updater()
+        updater.wait_healthy = lambda _: False
+        self.assertEqual(updater.run(), "failed")
+        self.assert_unchanged()
+        self.assertEqual(json.loads(module.UPDATE_JOURNAL.read_text())["phase"], "rollback")
+        self.assertTrue((module.UPDATE_DIR / "agent.backup.py").exists())
+        self.assertIn("recovery is pending", self.status()["error"])
+        # A subsequent worker can safely confirm the restored release.
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+
+    def test_stale_wrong_pid_wrong_release_or_wrong_update_health_is_rejected(self):
+        updater = self.updater()
+        journal = {"id": "this-update", "targetVersion": "1.1.0", "switchedAt": time.time()}
+        valid = {"version": "1.1.0", "pid": self.pid, "checkedAt": time.time(), "updateId": "this-update"}
+        for key, value in (("pid", self.pid + 1), ("version", "1.0.0"), ("updateId", "previous-update"), ("checkedAt", journal["switchedAt"] - 1), ("checkedAt", float("nan")), ("checkedAt", time.time() + 100)):
+            with self.subTest(key=key, value=value):
+                module.AGENT_HEALTH.write_text(json.dumps({**valid, key: value}))
+                with patch.object(module, "HEALTH_TIMEOUT", 1), patch.object(module.time, "monotonic", side_effect=[0, 0, 2]), patch.object(module.time, "sleep"):
+                    self.assertFalse(updater.wait_healthy(journal))
+        module.AGENT_HEALTH.write_text(json.dumps(valid))
+        self.assertTrue(updater.wait_healthy(journal))
+
+    def test_symlinked_executable_is_rejected_without_touching_target(self):
+        external = self.base / "external.py"
+        external.write_bytes(PREVIOUS)
+        module.AGENT_EXECUTABLE.unlink()
+        module.AGENT_EXECUTABLE.symlink_to(external)
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertEqual(external.read_bytes(), PREVIOUS)
+        self.assertEqual(self.commands, [])
+
+    def test_network_reader_enforces_header_and_stream_bounds(self):
+        updater = module.Updater()
+        updater.load_installation()
+        for response in (Response(b"abc", {"Content-Length": "100"}), Response(b"abcdef")):
+            with self.subTest(response=response), patch.object(module.urllib.request, "build_opener") as opener:
+                opener.return_value.open.return_value = response
+                with self.assertRaises(module.IntegrityError):
+                    updater.fetch("/api/agent/release/ec-agent.py", 5)
+
+    def test_trickling_or_truncated_download_defers_without_holding_release(self):
+        updater = module.Updater()
+        updater.load_installation()
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            opener.return_value.open.return_value = Response(b"source")
+            with patch.object(module.time, "monotonic", side_effect=[0, 0, 121]):
+                with self.assertRaisesRegex(module.ContactError, "timed out"):
+                    updater.fetch("/api/agent/release/ec-agent.py", 100)
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            response = Response(b"source")
+            response.read1 = lambda *_: (_ for _ in ()).throw(module.http.client.IncompleteRead(b"partial"))
+            opener.return_value.open.return_value = response
+            with self.assertRaises(module.ContactError):
+                updater.fetch("/api/agent/release/ec-agent.py", 100)
+
+    def test_stale_code_staging_is_recovered_without_leaving_extra_executables(self):
+        temporary = module.AGENT_EXECUTABLE.with_name(module.AGENT_EXECUTABLE.name + ".update")
+        temporary.write_bytes(b"partial previous download")
+        self.assertEqual(self.updater().run(), "updated")
+        self.assertFalse(temporary.exists())
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+
+    def test_download_uses_fixed_https_controller_and_no_enrollment_credentials(self):
+        updater = module.Updater()
+        updater.load_installation()
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            opener.return_value.open.return_value = Response(b"source", {"Content-Length": "6"})
+            self.assertEqual(updater.fetch("/api/agent/release/ec-agent.py", 100), b"source")
+            request = opener.return_value.open.call_args.args[0]
+            self.assertEqual(request.full_url, "https://controller.example/api/agent/release/ec-agent.py")
+            self.assertIsNone(request.get_header("Authorization"))
+            redirect = opener.call_args.args[0]
+            with self.assertRaises(module.ContactError):
+                redirect.redirect_request(None, None, 302, "redirect", {}, "https://other.example")
+
+    def test_insecure_or_credential_embedded_controller_is_rejected(self):
+        for controller in ("http://controller.example", "https://token:secret@controller.example"):
+            with self.subTest(controller=controller):
+                module.AGENT_CONFIG.write_text(json.dumps({**self.config, "controller": controller}))
+                self.assertEqual(self.updater().run(), "failed")
+                self.assertEqual(self.downloads, [])
+                self.assertEqual(self.commands, [])
+
+
+class UpdateSecurityTests(unittest.TestCase):
+    def test_untrusted_ownership_writable_paths_and_special_files_are_rejected(self):
+        for mode, uid in ((stat.S_IFREG | 0o666, 0), (stat.S_IFDIR | 0o755, 1000), (stat.S_IFLNK | 0o777, 0), (stat.S_IFIFO | 0o600, 0)):
+            with self.subTest(mode=mode, uid=uid), patch.object(Path, "lstat", return_value=os.stat_result((mode, 1, 1, 1, uid, 0, 0, 0, 0, 0))):
+                with self.assertRaises(module.UpdateError):
+                    module.trusted("/etc/electric-cherry/update/journal.json")
+
+    def test_source_is_parsed_and_compiled_without_execution(self):
+        source = b'VERSION="1.2.3"\nraise RuntimeError("Never execute downloaded code during verification")\n'
+        self.assertEqual(module.source_version(source), "1.2.3")
+
+    def test_numeric_version_order_and_strict_format(self):
+        self.assertGreater(module.version("1.10.0"), module.version("1.9.9"))
+        for value in ("v1.0.0", "1.0", "1.2.03", "1.0.0+local", "1.0.0-rc1", True, None, "1.0.000000000000"):
+            with self.subTest(value=value), self.assertRaises(module.IntegrityError):
+                module.version(value)
+
+
+if __name__ == "__main__":
+    unittest.main()