Add automatic agent updates with verified rollback

README.md

@@ -33,7 +33,7 @@ including SQLite metadata and published game-file storage.
    folder, including its DLC files. Name the library and version, then publish it.
    Uploads work before any host is enrolled.
 2. **Hosts → Add Host**: configure a Debian 12/13 amd64 machine and run its
-   generated enrollment command.
+   generated enrollment command. Automatic agent updates are enabled by default.
 3. Prepare a Plutainer runtime from the host page and download the published
    game-file version to that host from **Game Files**.
 4. Add the exact Plutonium server keys in **Keys**, or supply missing keys when
@@ -44,6 +44,13 @@ including SQLite metadata and published game-file storage.
 See [Native host management](docs/native-hosts.md) for installation, credentials,
 storage, failure recovery, API contracts and the live-host acceptance checklist.
 
+Existing hosts need the current installer once to enable automatic updates.
+Generate it under **Agent access** and rerun it on the same host without
+`--reenroll`; identity and game data are preserved. Thereafter hosts check their
+controller about every five minutes, verify compatible releases and update agent
+code between operations. Failed updates roll back; running game containers
+continue. The host page shows update status and the last check.
+
 ## Fleet and operations
 
 The fleet registry lives in SQLite, independently of host filesystems. Status
@@ -127,7 +134,7 @@ npm run test:agent
 npm run build
 ```
 
-Builds regenerate the downloadable agent and checksum manifest. Local controller
+Builds regenerate the downloadable agent, standalone updater and checksum manifest. Local controller
 metadata defaults to `./data/cherry.db`; use a separate test database when
 exercising enrollment. Production agent transport requires HTTPS.
 

agent/ec_agent.py

@@ -29,7 +29,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.1.0"
+VERSION = "1.2.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
@@ -39,6 +39,13 @@ AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service")
 UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
 UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service")
 UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
+UPDATE_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-update.py")
+UPDATE_UNIT = Path("/etc/systemd/system/electric-cherry-update.service")
+UPDATE_TIMER = Path("/etc/systemd/system/electric-cherry-update.timer")
+UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-cherry-update.timer")
+UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
+UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
+AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
 ROOT_MARKER = ".electric-cherry.json"
 
 
@@ -105,7 +112,7 @@ def uninstall_root(value, identity=None):
     forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"}
     if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden:
         raise AgentError("Host data directory is unsafe to remove")
-    for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT):
+    for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT, UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER, AGENT_EXECUTABLE.with_suffix(".py.update"), UNINSTALL_EXECUTABLE.with_suffix(".py.update")):
         if installed == root or root in installed.parents:
             raise AgentError("Host data directory overlaps the agent installation")
     protected_path(root, missing=True)
@@ -499,6 +506,19 @@ class Uninstaller:
 
     def remove_data(self):
         protected_path(AGENT_CONFIG.parent, missing=True)
+        # Prevent a scheduled update from recreating agent files during removal.
+        # Stopping the worker is safe here: the installation and any backups are
+        # being removed together, and the durable decommission request blocks
+        # future update attempts.
+        if not self.job.get("updaterStopped"):
+            for unit in (UPDATE_TIMER, UPDATE_UNIT):
+                protected_file(unit, missing=True)
+                if unit.exists():
+                    if unit == UPDATE_TIMER:
+                        systemctl("disable", "--now", unit.name)
+                    else:
+                        systemctl("stop", unit.name)
+            self.save(updaterStopped=True)
         if not self.job.get("agentStopped"):
             systemctl("disable", "--now", AGENT_UNIT.name)
             self.save(agentStopped=True)
@@ -512,7 +532,7 @@ class Uninstaller:
                 raise AgentError("This platform cannot safely remove managed host files")
             shutil.rmtree(root)
             sync_parent(root)
-        for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT):
+        for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER, AGENT_EXECUTABLE.with_suffix(".py.update"), UNINSTALL_EXECUTABLE.with_suffix(".py.update")):
             protected_file(path, missing=True)
             path.unlink(missing_ok=True)
             sync_parent(path)
@@ -591,6 +611,74 @@ class Agent:
         except (AgentError, OSError, ValueError):
             return False
 
+    def can_update(self):
+        try:
+            if Path(self.config_path) != AGENT_CONFIG:
+                return False
+            uninstall_ownership(self.config)
+            for path in (UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER):
+                protected_file(path)
+            protected_path(UPDATE_TIMER_ENABLED.parent)
+            return UPDATE_TIMER_ENABLED.is_symlink() and UPDATE_TIMER_ENABLED.resolve() == UPDATE_TIMER
+        except (AgentError, OSError, ValueError):
+            return False
+
+    def update_inventory(self):
+        result = {"enabled": self.can_update(), "state": "current", "currentVersion": VERSION}
+        if Path(self.config_path) != AGENT_CONFIG:
+            return result
+        try:
+            protected_file(UPDATE_STATUS, missing=True)
+            if not UPDATE_STATUS.exists():
+                return result
+            if UPDATE_STATUS.stat().st_size > 16384:
+                raise ValueError("Invalid update status")
+            saved = json.loads(UPDATE_STATUS.read_text())
+            if saved.get("state") in ("current", "checking", "deferred", "updating", "updated", "failed"):
+                result["state"] = saved["state"]
+            for key in ("targetVersion", "error"):
+                if isinstance(saved.get(key), str):
+                    result[key] = saved[key][:2000 if key == "error" else 80]
+            checked_at = saved.get("checkedAt")
+            if isinstance(checked_at, (int, float)) and not isinstance(checked_at, bool):
+                result["checkedAt"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(checked_at))
+            elif isinstance(checked_at, str):
+                result["checkedAt"] = checked_at[:80]
+        except (AgentError, OSError, ValueError, TypeError, AttributeError, OverflowError):
+            result.update(state="failed", error="Agent update status is unavailable; inspect the update service.")
+        return result
+
+    def update_pending(self):
+        # A persisted transaction survives crashes and is cleared only by the
+        # independent worker after health confirmation or a completed rollback.
+        return Path(self.config_path) == AGENT_CONFIG and (UPDATE_JOURNAL.exists() or UPDATE_JOURNAL.is_symlink())
+
+    @contextlib.contextmanager
+    def activity(self):
+        path = self.root / "agent" / "activity.lock"
+        descriptor = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+        try:
+            try:
+                fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError:
+                yield False
+                return
+            yield True
+        finally:
+            os.close(descriptor)
+
+    def record_health(self):
+        if Path(self.config_path) != AGENT_CONFIG or not self.can_update():
+            return
+        value = {"version": VERSION, "pid": os.getpid(), "checkedAt": time.time()}
+        protected_file(AGENT_HEALTH, missing=True)
+        if self.update_pending():
+            protected_file(UPDATE_JOURNAL)
+            journal = json.loads(UPDATE_JOURNAL.read_text())
+            if isinstance(journal.get("id"), str):
+                value["updateId"] = journal["id"]
+        atomic_json(AGENT_HEALTH, value)
+
     def decommission(self, operation):
         payload = operation.get("payload", {})
         if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]:
@@ -733,7 +821,9 @@ class Agent:
             for line in Path("/etc/os-release").read_text().splitlines():
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
-        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
+        update = self.update_inventory()
+        capabilities = (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
         health = self.inspect(instance_id, generation)
@@ -1462,29 +1552,21 @@ class Agent:
         host = response.get("host", {})
         if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int):
             self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]}
+        # Written only after the controller accepted this process's authenticated
+        # heartbeat. Service liveness alone cannot prove an update works.
+        self.record_health()
 
     def heartbeat_loop(self):
         while not self.stopping.wait(15):
             try:
                 self.heartbeat()
-            except AgentError:
+            except (AgentError, OSError, ValueError):
                 pass
 
-    def run(self):
-        if self.config.get("pendingCredential"):
-            self.recover_credential()
-        # Obtain the configured host port range before accepting any deployment.
-        # Enrollment's default values must never race the first heartbeat.
-        while not self.stopping.is_set():
-            try:
-                self.heartbeat()
-                break
-            except AgentError:
-                self.stopping.wait(3)
-        heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
-        heartbeats.start()
-        while not self.stopping.is_set():
-            try:
+    def poll_operations(self):
+        with self.activity() as acquired:
+            if not acquired or self.stopping.is_set() or self.update_pending():
+                return
             if self.config.get("pendingCredential"):
                 self.recover_credential()
             response = self.request("GET", "/api/agent/operations")
@@ -1492,10 +1574,33 @@ class Agent:
                 if self.stopping.is_set():
                     break
                 self.current_operation = operation["id"]
+                try:
                     accepted, _ = self.journal.accept(operation)
                     self.report(accepted)
                     self.report(self.execute(operation, report_running=True))
+                finally:
                     self.current_operation = None
+
+    def run(self):
+        # Obtain the configured host port range before accepting any deployment.
+        # Enrollment's default values must never race the first heartbeat.
+        while not self.stopping.is_set():
+            try:
+                with self.activity() as acquired:
+                    if not acquired:
+                        self.stopping.wait(3)
+                        continue
+                    if self.config.get("pendingCredential"):
+                        self.recover_credential()
+                self.heartbeat()
+                break
+            except (AgentError, OSError, ValueError):
+                self.stopping.wait(3)
+        heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
+        heartbeats.start()
+        while not self.stopping.is_set():
+            try:
+                self.poll_operations()
             except (AgentError, OSError, ValueError, KeyError):
                 # Logs deliberately omit URLs, tokens, payloads and raw errors.
                 print("Host controller contact unavailable; existing containers continue running", file=sys.stderr)

agent/ec_agent_update.py

@@ -0,0 +1,479 @@
+#!/usr/bin/env python3
+"""Stable Electric Cherry agent updater; only verified agent code is replaced.
+
+The installer owns this bootstrap and its systemd timer. Updates never execute
+downloaded code before installing it, alter game containers, or edit enrollment
+credentials. A durable journal makes an interrupted switch roll back safely.
+"""
+
+import ast
+import contextlib
+import fcntl
+import hashlib
+import http.client
+import json
+import math
+import os
+from pathlib import Path
+import re
+import stat
+import subprocess
+import sys
+import time
+import urllib.parse
+import urllib.request
+import uuid
+
+UPDATER_PROTOCOL_VERSION = 1
+AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
+AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
+UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
+UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
+UPDATE_LOCK = Path("/etc/electric-cherry/update.lock")
+UPDATE_DIR = Path("/etc/electric-cherry/update")
+UPDATE_JOURNAL = UPDATE_DIR / "journal.json"
+UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
+AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
+AGENT_SERVICE = "electric-cherry-agent.service"
+MAX_MANIFEST = 16 * 1024
+MAX_SOURCE = 8 * 1024 * 1024
+HEALTH_TIMEOUT = 90
+VERSION_RE = re.compile(r"^(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})$")
+DIGEST_RE = re.compile(r"^[0-9a-f]{64}$")
+
+
+class UpdateError(Exception):
+    pass
+
+
+class IntegrityError(UpdateError):
+    pass
+
+
+class ContactError(UpdateError):
+    pass
+
+
+class Deferred(UpdateError):
+    pass
+
+
+def trusted(path, missing=False, directory=False):
+    path = Path(path)
+    for current in (*reversed(path.parents), path):
+        try:
+            info = current.lstat()
+        except FileNotFoundError:
+            if missing:
+                continue
+            raise UpdateError("Agent update files are unavailable; repair the installation") from None
+        expected_type = stat.S_ISDIR if current != path or directory else stat.S_ISREG
+        if not expected_type(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+            raise UpdateError("Agent update paths must be regular root-owned files and directories that other users cannot modify")
+    return path
+
+
+def sync_parent(path):
+    descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def atomic_bytes(path, value, mode=0o600):
+    path = trusted(path, missing=True)
+    trusted(path.parent, directory=True)
+    # Code staging uses known names so recovery and host removal can also
+    # remove a file left behind by a power loss during the atomic write.
+    temporary = path.with_name(path.name + ".update") if path in (AGENT_EXECUTABLE, UNINSTALL_EXECUTABLE) else path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp")
+    try:
+        if temporary.exists() or temporary.is_symlink():
+            trusted(temporary)
+            temporary.unlink()
+            sync_parent(temporary)
+        descriptor = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, mode)
+        with os.fdopen(descriptor, "wb") as stream:
+            stream.write(value)
+            stream.flush()
+            os.fsync(stream.fileno())
+        trusted(path, missing=True)
+        os.replace(temporary, path)
+        sync_parent(path)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def atomic_json(path, value):
+    atomic_bytes(path, json.dumps(value, sort_keys=True, allow_nan=False).encode())
+
+
+def read_bytes(path, limit):
+    trusted(path)
+    with open(path, "rb") as stream:
+        value = stream.read(limit + 1)
+    if len(value) > limit:
+        raise UpdateError("An agent update file exceeds its size limit")
+    return value
+
+
+def read_json(path, limit=MAX_MANIFEST):
+    try:
+        value = json.loads(read_bytes(path, limit))
+    except (ValueError, UnicodeError):
+        raise UpdateError("Agent update metadata is invalid") from None
+    if not isinstance(value, dict):
+        raise UpdateError("Agent update metadata is invalid")
+    return value
+
+
+def sha256(value):
+    return hashlib.sha256(value).hexdigest()
+
+
+def version(value):
+    if not isinstance(value, str) or not VERSION_RE.fullmatch(value):
+        raise IntegrityError("Agent releases must use a numeric major.minor.patch version")
+    return tuple(int(part) for part in value.split("."))
+
+
+def source_version(source):
+    try:
+        tree = ast.parse(source, filename="electric-cherry-agent.py")
+        compile(tree, "electric-cherry-agent.py", "exec")
+    except (SyntaxError, ValueError, TypeError, RecursionError, MemoryError):
+        raise IntegrityError("The agent release is not valid Python source") from None
+    declarations = [node.value.value for node in tree.body if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == "VERSION" for target in node.targets) and isinstance(node.value, ast.Constant)]
+    if len(declarations) != 1:
+        raise IntegrityError("The agent release must declare one explicit VERSION")
+    version(declarations[0])
+    return declarations[0]
+
+
+@contextlib.contextmanager
+def exclusive_lock(path):
+    trusted(path, missing=True)
+    trusted(Path(path).parent, directory=True)
+    descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
+    with os.fdopen(descriptor, "r+") as stream:
+        try:
+            fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            yield None
+            return
+        try:
+            yield stream
+        finally:
+            fcntl.flock(stream, fcntl.LOCK_UN)
+
+
+def systemctl(*args):
+    try:
+        result = subprocess.run(["systemctl", *args], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=60, check=False)
+    except (OSError, subprocess.TimeoutExpired):
+        raise UpdateError("Agent service management is unavailable") from None
+    if result.returncode:
+        raise UpdateError("Agent service management did not complete")
+    return result.stdout.decode("utf-8", "replace")
+
+
+def service_state():
+    output = systemctl("show", AGENT_SERVICE, "--property=ActiveState", "--property=MainPID")
+    fields = dict(line.split("=", 1) for line in output.splitlines() if "=" in line)
+    try:
+        return fields.get("ActiveState") == "active", int(fields.get("MainPID", "0"))
+    except ValueError:
+        return False, 0
+
+
+class Updater:
+    def __init__(self):
+        self.installation = None
+        self.current_version = None
+        self.target = None
+
+    def load_installation(self):
+        config = read_json(AGENT_CONFIG, 64 * 1024)
+        controller = config.get("controller")
+        root_value = config.get("root")
+        host_id = config.get("hostId")
+        if not isinstance(controller, str) or not isinstance(root_value, str) or not isinstance(host_id, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,127}", host_id):
+            raise UpdateError("Agent enrollment is incomplete")
+        parsed = urllib.parse.urlsplit(controller)
+        if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise UpdateError("Agent updates require an HTTPS controller without embedded credentials")
+        root = Path(root_value)
+        if not re.fullmatch(r"/[A-Za-z0-9/_.-]+", root_value) or str(root) != root_value or ".." in root.parts or len(root.parts) < 3:
+            raise UpdateError("The enrolled agent data directory is invalid")
+        trusted(root, directory=True)
+        trusted(root / "agent", directory=True)
+        marker = read_json(root / ".electric-cherry.json")
+        controller = controller.rstrip("/")
+        expected = {"version": 1, "root": root_value, "hostId": host_id, "controller": controller}
+        if marker != expected:
+            raise UpdateError("Agent data ownership does not match this enrollment")
+        info = root.stat()
+        installation = {"hostId": host_id, "controller": controller, "root": root_value, "rootIdentity": [info.st_dev, info.st_ino]}
+        if self.installation is not None and installation != self.installation:
+            raise UpdateError("Agent enrollment changed during the update; no code was replaced")
+        self.installation = installation
+        return installation
+
+    def ensure_directory(self):
+        trusted(UPDATE_DIR, missing=True, directory=True)
+        trusted(UPDATE_DIR.parent, directory=True)
+        if not UPDATE_DIR.exists():
+            UPDATE_DIR.mkdir(mode=0o700)
+            sync_parent(UPDATE_DIR)
+
+    def check_uninstall(self):
+        if UNINSTALL_REQUEST.exists() or UNINSTALL_REQUEST.is_symlink():
+            raise Deferred("Host deletion is pending; agent updates are paused")
+
+    def status(self, state, error=None, current=None):
+        value = {"state": state, "checkedAt": time.time()}
+        if current or self.current_version:
+            value["currentVersion"] = current or self.current_version
+        if self.target:
+            value["targetVersion"] = self.target["version"]
+        if error:
+            value["error"] = str(error)[:500]
+        atomic_json(UPDATE_STATUS, value)
+        return state
+
+    def fetch(self, path, limit):
+        request = urllib.request.Request(self.installation["controller"] + path, headers={"User-Agent": "electric-cherry-updater/1"})
+        class NoRedirect(urllib.request.HTTPRedirectHandler):
+            def redirect_request(self, req, fp, code, msg, headers, newurl):
+                raise ContactError("Agent release redirects are not allowed")
+        try:
+            with urllib.request.build_opener(NoRedirect()).open(request, timeout=25) as response:
+                if response.status != 200:
+                    raise ContactError("The controller did not return a complete agent release")
+                length = response.headers.get("Content-Length")
+                if length is not None and (not length.isdecimal() or int(length) > limit):
+                    raise IntegrityError("The controller release exceeds its download size limit")
+                chunks, total = [], 0
+                deadline = time.monotonic() + 120
+                while True:
+                    if time.monotonic() >= deadline:
+                        raise ContactError("The controller release download timed out")
+                    # HTTPResponse.read1 performs one socket read. read(n) can
+                    # wait indefinitely for n bytes from a trickling server.
+                    block = response.read1(min(64 * 1024, limit + 1 - total))
+                    if not block:
+                        break
+                    total += len(block)
+                    if total > limit:
+                        raise IntegrityError("The controller release exceeds its download size limit")
+                    chunks.append(block)
+                if length is not None and total != int(length):
+                    raise ContactError("The controller release download was interrupted")
+                return b"".join(chunks)
+        except (OSError, ValueError, http.client.HTTPException):
+            raise ContactError("The controller release is unavailable; the running agent was left unchanged") from None
+
+    def manifest(self):
+        try:
+            value = json.loads(self.fetch("/api/agent/release/manifest.json", MAX_MANIFEST))
+        except (ValueError, UnicodeError):
+            raise IntegrityError("The controller release manifest is invalid") from None
+        if not isinstance(value, dict) or type(value.get("protocolVersion")) is not int or value["protocolVersion"] != 1 or value.get("platform") != "debian-amd64":
+            raise IntegrityError("The controller release is not compatible with this host agent")
+        if type(value.get("updaterProtocolVersion")) is not int or value["updaterProtocolVersion"] != UPDATER_PROTOCOL_VERSION:
+            raise Deferred("This release needs a newer updater bootstrap; rerun the host installer")
+        version(value.get("version"))
+        if not isinstance(value.get("sha256"), str) or not DIGEST_RE.fullmatch(value["sha256"]):
+            raise IntegrityError("The controller release manifest has an invalid checksum")
+        return {"version": value["version"], "sha256": value["sha256"]}
+
+    def hold(self, target, error):
+        self.ensure_directory()
+        atomic_json(UPDATE_DIR / "failed-release.json", {**target, "error": error, "failedAt": time.time()})
+
+    def release_held(self):
+        path = UPDATE_DIR / "failed-release.json"
+        if not path.exists():
+            return False
+        held = read_json(path)
+        return all(held.get(key) == self.target[key] for key in ("version", "sha256"))
+
+    def save_journal(self, journal, **changes):
+        journal.update(changes)
+        atomic_json(UPDATE_JOURNAL, journal)
+
+    def read_journal(self):
+        journal = read_json(UPDATE_JOURNAL)
+        if journal.get("installation") != self.installation or journal.get("phase") not in ("installing", "verifying", "rollback") or not isinstance(journal.get("id"), str) or not re.fullmatch(r"[a-f0-9]{32}", journal["id"]):
+            raise UpdateError("Pending agent update ownership or recovery metadata is invalid")
+        version(journal.get("previousVersion"))
+        version(journal.get("targetVersion"))
+        for field in ("targetSha256", "agentBackupSha256", "uninstallBackupSha256"):
+            if not isinstance(journal.get(field), str) or not DIGEST_RE.fullmatch(journal[field]):
+                raise UpdateError("Pending agent update recovery checksums are invalid")
+        return journal
+
+    def finish_journal(self):
+        trusted(UPDATE_JOURNAL)
+        UPDATE_JOURNAL.unlink()
+        sync_parent(UPDATE_JOURNAL)
+        # Removing the journal commits the decision. Backups are disposable
+        # only afterward, so a crash never leaves recovery without its inputs.
+        for name in ("agent.backup.py", "uninstall.backup.py", "candidate.py"):
+            path = trusted(UPDATE_DIR / name, missing=True)
+            path.unlink(missing_ok=True)
+        sync_parent(UPDATE_JOURNAL)
+
+    def activity_lock(self):
+        return exclusive_lock(Path(self.installation["root"]) / "agent" / "activity.lock")
+
+    def install(self, source):
+        with self.activity_lock() as activity:
+            if activity is None:
+                raise Deferred("The agent is executing an operation; update will retry when it is idle")
+            self.check_uninstall()
+            self.load_installation()
+            active, pid = service_state()
+            if not active or pid <= 0:
+                raise Deferred("Agent service is not running; automatic updates wait until it is started")
+            previous = read_bytes(AGENT_EXECUTABLE, MAX_SOURCE)
+            uninstall = read_bytes(UNINSTALL_EXECUTABLE, MAX_SOURCE)
+            if source_version(previous) != self.current_version:
+                raise Deferred("The installed agent changed while checking for updates")
+            source_version(uninstall)
+            self.ensure_directory()
+            atomic_bytes(UPDATE_DIR / "agent.backup.py", previous)
+            atomic_bytes(UPDATE_DIR / "uninstall.backup.py", uninstall)
+            atomic_bytes(UPDATE_DIR / "candidate.py", source)
+            journal = {"id": uuid.uuid4().hex, "phase": "installing", "installation": self.installation, "previousVersion": self.current_version, "targetVersion": self.target["version"], "targetSha256": self.target["sha256"], "agentBackupSha256": sha256(previous), "uninstallBackupSha256": sha256(uninstall), "startedAt": time.time()}
+            self.save_journal(journal)
+            self.status("updating")
+            self.check_uninstall()
+            self.load_installation()
+            systemctl("stop", AGENT_SERVICE)
+            atomic_bytes(AGENT_EXECUTABLE, source, 0o755)
+            atomic_bytes(UNINSTALL_EXECUTABLE, source, 0o755)
+            self.save_journal(journal, phase="verifying", switchedAt=time.time())
+        # The candidate takes the same lock during startup credential recovery.
+        # Never wait for its start or authenticated heartbeat while holding it.
+        systemctl("start", AGENT_SERVICE)
+        if not self.wait_healthy(journal):
+            raise UpdateError("The new agent did not report a healthy authenticated heartbeat; the previous release will be restored")
+        self.check_uninstall()
+        self.load_installation()
+        if any(sha256(read_bytes(path, MAX_SOURCE)) != journal["targetSha256"] for path in (AGENT_EXECUTABLE, UNINSTALL_EXECUTABLE)):
+            raise UpdateError("Installed agent code changed before update verification finished")
+        self.status("updated", current=journal["targetVersion"])
+        self.current_version = journal["targetVersion"]
+        self.finish_journal()
+        return "updated"
+
+    def wait_healthy(self, journal):
+        deadline = time.monotonic() + HEALTH_TIMEOUT
+        while time.monotonic() < deadline:
+            self.check_uninstall()
+            try:
+                active, pid = service_state()
+                health = read_json(AGENT_HEALTH, 4096)
+                checked = health.get("checkedAt")
+                if active and pid > 0 and type(health.get("pid")) is int and health["pid"] == pid and health.get("version") == journal["targetVersion"] and health.get("updateId") == journal["id"] and type(checked) in (float, int) and math.isfinite(checked) and journal["switchedAt"] <= checked <= time.time() + 5:
+                    return True
+            except (UpdateError, OSError, ValueError):
+                pass
+            time.sleep(2)
+        return False
+
+    def rollback(self, journal, error):
+        self.target = {"version": journal["targetVersion"], "sha256": journal["targetSha256"]}
+        self.check_uninstall()
+        self.load_installation()
+        if journal["installation"] != self.installation:
+            raise UpdateError("Agent enrollment changed; rollback needs operator review")
+        backups = [(AGENT_EXECUTABLE, "agent.backup.py", "agentBackupSha256"), (UNINSTALL_EXECUTABLE, "uninstall.backup.py", "uninstallBackupSha256")]
+        sources = []
+        for destination, name, checksum in backups:
+            source = read_bytes(UPDATE_DIR / name, MAX_SOURCE)
+            if sha256(source) != journal[checksum]:
+                raise UpdateError("Agent rollback backup failed integrity verification; repair the installation")
+            source_version(source)
+            sources.append((destination, source))
+        if source_version(sources[0][1]) != journal["previousVersion"]:
+            raise UpdateError("Agent rollback version does not match its recovery journal")
+        self.save_journal(journal, phase="rollback")
+        with self.activity_lock() as activity:
+            if activity is None:
+                raise Deferred("Agent rollback is waiting for the active operation to finish")
+            self.check_uninstall()
+            self.load_installation()
+            systemctl("stop", AGENT_SERVICE)
+            for destination, source in sources:
+                atomic_bytes(destination, source, 0o755)
+            self.save_journal(journal, rollbackSwitchedAt=time.time())
+        systemctl("start", AGENT_SERVICE)
+        rollback_health = {**journal, "targetVersion": journal["previousVersion"], "switchedAt": journal["rollbackSwitchedAt"]}
+        if not self.wait_healthy(rollback_health):
+            raise UpdateError("The previous agent release was restored but its authenticated health is unconfirmed; recovery will retry")
+        self.hold(self.target, error)
+        self.status("failed", error, current=journal["previousVersion"])
+        self.current_version = journal["previousVersion"]
+        self.finish_journal()
+        return "failed"
+
+    def run(self):
+        with exclusive_lock(UPDATE_LOCK) as updater:
+            if updater is None:
+                return "deferred"  # Preserve the active worker's status.
+            recovering = False
+            try:
+                self.check_uninstall()
+                self.load_installation()
+                if UPDATE_JOURNAL.exists() or UPDATE_JOURNAL.is_symlink():
+                    journal = self.read_journal()
+                    self.current_version = journal["previousVersion"]
+                    recovering = True
+                    return self.rollback(journal, "An interrupted agent update was rolled back to the previous release")
+                self.current_version = source_version(read_bytes(AGENT_EXECUTABLE, MAX_SOURCE))
+                self.status("checking")
+                self.target = self.manifest()
+                if version(self.target["version"]) <= version(self.current_version):
+                    return self.status("current")
+                if self.release_held():
+                    return self.status("failed", "This release previously failed verification and is held until the controller publishes a different release")
+                source = self.fetch("/api/agent/release/ec-agent.py", MAX_SOURCE)
+                if sha256(source) != self.target["sha256"]:
+                    raise IntegrityError("The agent release checksum does not match its manifest")
+                if source_version(source) != self.target["version"]:
+                    raise IntegrityError("The agent source VERSION does not match its release manifest")
+                return self.install(source)
+            except Deferred as error:
+                return self.status("deferred", error)
+            except ContactError as error:
+                return self.status("deferred", error)
+            except (UpdateError, OSError, ValueError, KeyError) as error:
+                message = str(error) if isinstance(error, UpdateError) else "Agent update could not finish; inspect the updater service"
+                if UPDATE_JOURNAL.exists():
+                    if recovering:
+                        return self.status("failed", "Agent update recovery is pending; repair the updater service before accepting new operations")
+                    try:
+                        return self.rollback(self.read_journal(), message)
+                    except Deferred as deferred:
+                        return self.status("deferred", deferred)
+                    except (UpdateError, OSError, ValueError, KeyError):
+                        return self.status("failed", "Agent update recovery is pending; repair the updater service before accepting new operations")
+                if isinstance(error, IntegrityError) and self.target:
+                    self.hold(self.target, message)
+                return self.status("failed", message)
+
+
+def main():
+    if os.geteuid() != 0:
+        raise UpdateError("Run the agent updater through its installed root service")
+    Updater().run()
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (UpdateError, OSError, ValueError, KeyError):
+        print("Electric Cherry agent updater failed; inspect its installation and update status", file=sys.stderr)
+        sys.exit(1)

agent/install.sh

@@ -8,6 +8,8 @@ controller=''
 enrollment_token=''
 data_dir='/var/lib/electric-cherry'
 reenroll=''
+existing_enrollment=false
+existing_data_root=false
 while [ "$#" -gt 0 ]; do
     case "$1" in
         --controller) controller="${2:?Missing controller URL}"; shift 2 ;;
@@ -31,7 +33,16 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; }
 
 export DEBIAN_FRONTEND=noninteractive
 apt-get update -qq
-apt-get install -y --no-install-recommends ca-certificates curl python3
+apt-get install -y --no-install-recommends ca-certificates curl python3 util-linux
+installation_preflight() {
+    if [ "$existing_enrollment" = true ] && [ ! -f /etc/electric-cherry/agent.json ]; then
+        echo 'The existing enrollment disappeared during installation. Stop and inspect host deletion before trying again.' >&2
+        return 1
+    fi
+    if [ "$existing_data_root" = true ] && [ ! -d "$data_dir" ]; then
+        echo 'The existing data directory disappeared during installation. Stop and inspect host deletion before trying again.' >&2
+        return 1
+    fi
 python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY'
 # INSTALLATION_PREFLIGHT: this runs before touching managed directories or services.
 import json, pathlib, re, stat, sys
@@ -51,9 +62,14 @@ if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root)
     raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.')
 installed_paths = [config,
     pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-agent.py.update'),
     pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py.update'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-update.py'),
     pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'),
-    pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service')]
+    pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-update.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-update.timer')]
 if any(root == path or root in path.parents for path in installed_paths):
     raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.')
 
@@ -66,10 +82,13 @@ def trusted_directories(path):
         if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
             raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.')
 
-def trusted_json(path):
+def trusted_file(path):
     info = path.lstat()
     if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
         raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.')
+
+def trusted_json(path):
+    trusted_file(path)
     value = json.loads(path.read_text())
     if not isinstance(value, dict):
         raise SystemExit('Existing installation metadata is invalid.')
@@ -89,6 +108,13 @@ if mountinfo.exists():
 request = config.with_name('decommission.json')
 if request.exists() or request.is_symlink():
     raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.')
+trusted_directories(config.parent / 'update')
+journal = config.parent / 'update' / 'journal.json'
+if journal.exists() or journal.is_symlink():
+    raise SystemExit('An automatic update needs recovery. Let electric-cherry-update.service finish or repair that update before rerunning the installer.')
+update_lock = config.with_name('update.lock')
+if update_lock.exists() or update_lock.is_symlink():
+    trusted_file(update_lock)
 saved = None
 if config.exists() or config.is_symlink():
     saved = trusted_json(config)
@@ -108,6 +134,62 @@ elif root.exists() and any(root.iterdir()):
         raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.')
 for directory in ('agent', 'instances', 'libraries'):
     trusted_directories(root / directory)
+activity_lock = root / 'agent' / 'activity.lock'
+if activity_lock.exists() or activity_lock.is_symlink():
+    trusted_file(activity_lock)
+PY
+}
+if [ -f /etc/electric-cherry/agent.json ]; then existing_enrollment=true; fi
+if [ -d "$data_dir" ]; then existing_data_root=true; fi
+installation_preflight
+temporary_dir=$(mktemp -d)
+updater_timer_was_active=false
+update_lock_held=false
+activity_lock_held=false
+installer_finished=false
+installation_cleanup() {
+    local ec_install_status=$?
+    trap - EXIT
+    if [ "$activity_lock_held" = true ]; then
+        flock --unlock 8 || true
+        exec 8>&-
+    fi
+    if [ "$update_lock_held" = true ]; then
+        flock --unlock 9 || true
+        exec 9>&-
+    fi
+    # Recovery must remain scheduled when an upgrade or its second preflight
+    # fails. Release the worker lock before restarting the prior active timer.
+    if [ "$installer_finished" != true ] && [ "$updater_timer_was_active" = true ]; then
+        systemctl start electric-cherry-update.timer >/dev/null 2>&1 || true
+    fi
+    rm -rf "$temporary_dir" || true
+    exit "$ec_install_status"
+}
+trap installation_cleanup EXIT
+controller="${controller%/}"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent-update.py" -o "$temporary_dir/ec-agent-update.py"
+python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" "$temporary_dir/ec-agent-update.py" <<'PY'
+# INSTALLATION_RELEASE: check both downloads before installing or executing either.
+import ast, hashlib, json, pathlib, re, sys
+manifest = json.loads(pathlib.Path(sys.argv[1]).read_text())
+agent = pathlib.Path(sys.argv[2]).read_bytes()
+updater = pathlib.Path(sys.argv[3]).read_bytes()
+if hashlib.sha256(agent).hexdigest() != manifest.get('sha256'):
+    raise SystemExit('Agent release checksum verification failed.')
+if hashlib.sha256(updater).hexdigest() != manifest.get('updaterSha256'):
+    raise SystemExit('Updater release checksum verification failed.')
+if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1 or manifest.get('updaterProtocolVersion') != 1:
+    raise SystemExit('Unsupported agent or updater release manifest.')
+versions = [node.value.value for node in ast.parse(agent).body
+            if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == 'VERSION' for target in node.targets)
+            and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str)]
+if len(versions) != 1 or versions[0] != manifest.get('version') or not re.fullmatch(r'(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)', versions[0]):
+    raise SystemExit('Agent release version does not match its manifest.')
+compile(agent, 'ec-agent.py', 'exec')
+compile(updater, 'ec-agent-update.py', 'exec')
 PY
 if ! command -v docker >/dev/null; then
     if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then
@@ -118,29 +200,37 @@ if ! command -v docker >/dev/null; then
     systemctl enable --now docker
 fi
 docker info >/dev/null 2>&1 || { echo 'Docker is installed but unavailable. Start or repair it, then rerun.' >&2; exit 1; }
-install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances"
-install -d -m 0700 "$data_dir/agent" /etc/electric-cherry
-available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}')
-[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; }
-temporary_dir=$(mktemp -d)
-trap 'rm -rf "$temporary_dir"' EXIT
-controller="${controller%/}"
-curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json"
-curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py"
-python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" <<'PY'
-import hashlib, json, pathlib, sys
-manifest = json.loads(pathlib.Path(sys.argv[1]).read_text())
-actual = hashlib.sha256(pathlib.Path(sys.argv[2]).read_bytes()).hexdigest()
-if actual != manifest.get('sha256'):
-    raise SystemExit('Agent release checksum verification failed.')
-if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1:
-    raise SystemExit('Unsupported agent release manifest.')
-PY
+install -d -m 0700 /etc/electric-cherry
+# Wait for an already-running updater to finish; never interrupt its rollback.
+# The same lock also prevents a timer firing while files are being installed.
+if systemctl cat electric-cherry-update.timer >/dev/null 2>&1; then
+    if systemctl is-active --quiet electric-cherry-update.timer; then
+        updater_timer_was_active=true
+    fi
+    systemctl stop electric-cherry-update.timer
+fi
+exec 9>/etc/electric-cherry/update.lock
+flock --exclusive 9
+update_lock_held=true
+installation_preflight
+# Serialize with running operations, especially a deletion request that could
+# otherwise start between preflight and stopping the main agent.
+if [ -d "$data_dir/agent" ]; then
+    exec 8>"$data_dir/agent/activity.lock"
+    flock --exclusive 8
+    activity_lock_held=true
+    installation_preflight
+fi
 # Stop only the agent while replacing its own executable/journal owner. Existing
 # game containers keep running under Docker throughout installation and updates.
 if systemctl cat electric-cherry-agent.service >/dev/null 2>&1; then
     systemctl stop electric-cherry-agent.service
 fi
+installation_preflight
+install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances"
+install -d -m 0700 "$data_dir/agent"
+available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}')
+[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; }
 install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py 2>/dev/null || {
     install -d -m 0755 /usr/local/libexec
     install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py
@@ -148,6 +238,7 @@ install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-
 # A separate worker survives stopping/removing the agent service. Its only
 # entry point is a fixed, authenticated deletion request prepared by the agent.
 install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py
+install -m 0755 "$temporary_dir/ec-agent-update.py" /usr/local/libexec/electric-cherry-update.py
 printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token"
 unset enrollment_token
 python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"}
@@ -215,8 +306,41 @@ UMask=0077
 [Install]
 WantedBy=multi-user.target
 EOF
+cat > /etc/systemd/system/electric-cherry-update.service <<'EOF'
+[Unit]
+Description=Update the Electric Cherry Host Agent
+After=network-online.target docker.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-update.py
+TimeoutStartSec=0
+UMask=0077
+EOF
+cat > /etc/systemd/system/electric-cherry-update.timer <<'EOF'
+[Unit]
+Description=Check for Electric Cherry Host Agent Updates
+
+[Timer]
+OnBootSec=2min
+OnUnitInactiveSec=5min
+RandomizedDelaySec=30s
+AccuracySec=1s
+Unit=electric-cherry-update.service
+
+[Install]
+WantedBy=timers.target
+EOF
 systemctl daemon-reload
 systemctl enable electric-cherry-agent
+if [ "$activity_lock_held" = true ]; then
+    flock --unlock 8
+    exec 8>&-
+    activity_lock_held=false
+fi
 systemctl restart electric-cherry-agent
 systemctl is-active --quiet electric-cherry-agent || { echo 'Agent service did not start. Inspect journalctl -u electric-cherry-agent.' >&2; exit 1; }
-echo 'Electric Cherry host agent installed and enrolled. Host readiness is shown in Hosts.'
+systemctl enable --now electric-cherry-update.timer
+installer_finished=true
+echo 'Electric Cherry host agent installed and enrolled. Automatic agent updates are enabled; host readiness is shown in Hosts.'

agent/package_release.py

@@ -1,13 +1,37 @@
 #!/usr/bin/env python3
 """Publish the reviewed local agent source into the controller static release."""
+import ast
 import hashlib
 import json
 from pathlib import Path
+import re
 import shutil
 
-source = Path(__file__).resolve().parent
-target = source.parent / "static" / "agent"
-target.mkdir(parents=True, exist_ok=True)
-shutil.copyfile(source / "ec_agent.py", target / "ec-agent.py")
-shutil.copyfile(source / "install.sh", target / "install.sh")
-(target / "manifest.json").write_text(json.dumps({"version": "1.1.0", "protocolVersion": 1, "platform": "debian-amd64", "sha256": hashlib.sha256((target / "ec-agent.py").read_bytes()).hexdigest()}, indent=2) + "\n")
+
+def package(source, target):
+    """Read version metadata without importing or executing release code."""
+    source, target = Path(source), Path(target)
+    agent = (source / "ec_agent.py").read_bytes()
+    updater = (source / "ec_agent_update.py").read_bytes()
+    versions = [node.value.value for node in ast.parse(agent).body
+                if isinstance(node, ast.Assign) and any(isinstance(name, ast.Name) and name.id == "VERSION" for name in node.targets)
+                and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str)]
+    if len(versions) != 1 or not re.fullmatch(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", versions[0]):
+        raise ValueError("Agent source must declare exactly one literal semantic VERSION")
+    compile(agent, "ec_agent.py", "exec")
+    compile(updater, "ec_agent_update.py", "exec")
+    manifest = {"version": versions[0], "protocolVersion": 1, "platform": "debian-amd64",
+                "sha256": hashlib.sha256(agent).hexdigest(), "updaterProtocolVersion": 1,
+                "updaterSha256": hashlib.sha256(updater).hexdigest()}
+    target.mkdir(parents=True, exist_ok=True)
+    (target / "ec-agent.py").write_bytes(agent)
+    (target / "ec-agent-update.py").write_bytes(updater)
+    shutil.copyfile(source / "install.sh", target / "install.sh")
+    temporary_manifest = target / ".manifest.json.tmp"
+    temporary_manifest.write_text(json.dumps(manifest, indent=2) + "\n")
+    temporary_manifest.replace(target / "manifest.json")
+
+
+if __name__ == "__main__":
+    source = Path(__file__).resolve().parent
+    package(source, source.parent / "static" / "agent")

agent/tests/test_installer.py

@@ -1,10 +1,13 @@
 """Exercise installer guards without installing software or touching services."""
 
+import hashlib
+import importlib.util
 import json
 import os
 from pathlib import Path
 import re
 import stat
+import subprocess
 import sys
 import tempfile
 import unittest
@@ -15,6 +18,11 @@ SOURCE = (Path(__file__).resolve().parents[1] / "install.sh").read_text()
 BLOCKS = re.findall(r"<<'PY'\n(.*?)\nPY", SOURCE, re.DOTALL)
 PREFLIGHT = next(block for block in BLOCKS if "# INSTALLATION_PREFLIGHT:" in block)
 OWNERSHIP = next(block for block in BLOCKS if "# INSTALLATION_OWNERSHIP:" in block)
+RELEASE = next(block for block in BLOCKS if "# INSTALLATION_RELEASE:" in block)
+CLEANUP = re.search(r"installation_cleanup\(\) \{.*?\n\}", SOURCE, re.DOTALL).group()
+SPEC = importlib.util.spec_from_file_location("package_release", Path(__file__).resolve().parents[1] / "package_release.py")
+PACKAGER = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(PACKAGER)
 
 
 class InstallerTests(unittest.TestCase):
@@ -163,6 +171,47 @@ class InstallerTests(unittest.TestCase):
         with self.assertRaisesRegex(SystemExit, "deletion is in progress"):
             self.preflight()
 
+    def test_pending_update_recovery_is_preserved_and_blocks_manual_replacement(self):
+        self.legacy_installation()
+        update = self.config.parent / "update"
+        update.mkdir()
+        journal = update / "journal.json"
+        saved = '{"phase":"swapping","targetVersion":"1.2.0"}'
+        journal.write_text(saved)
+        with self.assertRaisesRegex(SystemExit, "automatic update needs recovery"):
+            self.preflight()
+        self.assertEqual(journal.read_text(), saved)
+        journal.unlink()
+        journal.symlink_to(self.base / "missing")
+        with self.assertRaisesRegex(SystemExit, "automatic update needs recovery"):
+            self.preflight()
+
+    def test_update_directory_and_lock_cannot_follow_symlinks(self):
+        self.legacy_installation()
+        elsewhere = self.base / "unrelated"
+        elsewhere.mkdir()
+        update = self.config.parent / "update"
+        update.symlink_to(elsewhere, target_is_directory=True)
+        with self.assertRaisesRegex(SystemExit, "not symlinks"):
+            self.preflight()
+        update.unlink()
+        lock = self.config.with_name("update.lock")
+        lock.symlink_to(elsewhere / "sentinel")
+        with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
+            self.preflight()
+        self.assertFalse((elsewhere / "sentinel").exists())
+        lock.unlink()
+        activity = self.root / "agent" / "activity.lock"
+        activity.symlink_to(elsewhere / "activity-sentinel")
+        with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
+            self.preflight()
+        self.assertFalse((elsewhere / "activity-sentinel").exists())
+
+    def test_data_root_cannot_contain_the_updater_or_service_installation(self):
+        for path in ("/etc/systemd", "/etc/systemd/system", "/usr/local/libexec/electric-cherry-update.py", "/usr/local/libexec/electric-cherry-agent.py.update", "/usr/local/libexec/electric-cherry-uninstall.py.update"):
+            with self.subTest(path=path), self.assertRaisesRegex(SystemExit, "outside the agent executable and service"):
+                self.preflight(path)
+
     def test_mounted_parent_is_allowed_but_data_root_and_nested_mounts_are_rejected(self):
         mountinfo = Path('/proc/1/mountinfo')
         original_exists, original_read = Path.exists, Path.read_text
@@ -181,5 +230,125 @@ class InstallerTests(unittest.TestCase):
                         self.preflight()
 
 
+class ReleaseTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.source = Path(self.temporary.name) / "source"
+        self.target = Path(self.temporary.name) / "release"
+        self.source.mkdir()
+        (self.source / "ec_agent.py").write_text('VERSION = "3.12.7"\nraise RuntimeError("must never execute during packaging")\n')
+        (self.source / "ec_agent_update.py").write_text('raise RuntimeError("must never execute during verification")\n')
+        (self.source / "install.sh").write_text("#!/bin/bash\n")
+        PACKAGER.package(self.source, self.target)
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def verify(self):
+        argv = ["-", str(self.target / "manifest.json"), str(self.target / "ec-agent.py"), str(self.target / "ec-agent-update.py")]
+        with patch.object(sys, "argv", argv):
+            exec(compile(RELEASE, "installer-release", "exec"), {})
+
+    def manifest(self, **changes):
+        path = self.target / "manifest.json"
+        value = json.loads(path.read_text())
+        value.update(changes)
+        path.write_text(json.dumps(value))
+        return value
+
+    def test_package_derives_version_and_hashes_both_sources_without_executing_them(self):
+        manifest = self.manifest()
+        self.assertEqual(manifest["version"], "3.12.7")
+        self.assertEqual(manifest["updaterProtocolVersion"], 1)
+        self.assertEqual(manifest["sha256"], hashlib.sha256((self.source / "ec_agent.py").read_bytes()).hexdigest())
+        self.assertEqual(manifest["updaterSha256"], hashlib.sha256((self.source / "ec_agent_update.py").read_bytes()).hexdigest())
+        self.assertEqual((self.target / "ec-agent-update.py").read_bytes(), (self.source / "ec_agent_update.py").read_bytes())
+        self.verify()
+
+    def test_changed_agent_and_updater_downloads_are_rejected(self):
+        for filename, error in (("ec-agent.py", "Agent release checksum"), ("ec-agent-update.py", "Updater release checksum")):
+            with self.subTest(filename=filename):
+                path = self.target / filename
+                original = path.read_bytes()
+                path.write_bytes(original + b"# changed\n")
+                with self.assertRaisesRegex(SystemExit, error):
+                    self.verify()
+                path.write_bytes(original)
+
+    def test_installer_rejects_missing_or_unsupported_updater_protocol(self):
+        for protocol in (None, 0, 2, "1"):
+            self.manifest(updaterProtocolVersion=protocol)
+            with self.subTest(protocol=protocol), self.assertRaisesRegex(SystemExit, "Unsupported agent or updater"):
+                self.verify()
+
+    def test_installer_rejects_manifest_version_mismatch_and_invalid_code(self):
+        self.manifest(version="3.12.8")
+        with self.assertRaisesRegex(SystemExit, "version does not match"):
+            self.verify()
+        malformed = b"def malformed(:\n"
+        (self.target / "ec-agent-update.py").write_bytes(malformed)
+        self.manifest(version="3.12.7", updaterSha256=hashlib.sha256(malformed).hexdigest())
+        with self.assertRaises(SyntaxError):
+            self.verify()
+
+    def test_packaging_requires_one_literal_semantic_agent_version(self):
+        for source in ('VERSION = "bad"', 'VERSION = "01.2.3"', 'VERSION = str(3)', 'VERSION = "1.2.3"\nVERSION = "1.2.4"'):
+            (self.source / "ec_agent.py").write_text(source)
+            with self.subTest(source=source), self.assertRaisesRegex(ValueError, "literal semantic VERSION"):
+                PACKAGER.package(self.source, self.target)
+
+    def test_installer_serializes_changes_and_enables_update_timer_only_after_agent_starts(self):
+        locked = SOURCE.index("flock --exclusive 9")
+        self.assertLess(SOURCE.index("systemctl stop electric-cherry-update.timer"), locked)
+        self.assertLess(locked, SOURCE.index("installation_preflight", locked))
+        self.assertLess(SOURCE.index("installation_preflight", locked), SOURCE.index('install -m 0755 "$temporary_dir/ec-agent.py"'))
+        self.assertLess(locked, SOURCE.index("flock --exclusive 8"))
+        self.assertLess(SOURCE.index("flock --exclusive 8"), SOURCE.index("systemctl stop electric-cherry-agent.service"))
+        self.assertLess(SOURCE.index("flock --unlock 8", SOURCE.index("systemctl daemon-reload")), SOURCE.index("systemctl restart electric-cherry-agent"))
+        self.assertLess(SOURCE.index("systemctl is-active --quiet electric-cherry-agent"), SOURCE.index("systemctl enable --now electric-cherry-update.timer"))
+        self.assertIn("ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-update.py", SOURCE)
+        self.assertIn("OnUnitInactiveSec=5min", SOURCE)
+        self.assertIn("RandomizedDelaySec=30s", SOURCE)
+        self.assertIn("AccuracySec=1s", SOURCE)
+        self.assertNotIn("Persistent=true", SOURCE)
+
+    def test_failed_install_restores_only_previously_active_timer_after_releasing_locks(self):
+        for previous, finished, code in ((True, False, 37), (False, False, 37), (True, True, 0)):
+            with self.subTest(previous=previous, finished=finished):
+                scratch = Path(self.temporary.name) / "scratch"
+                scratch.mkdir(exist_ok=True)
+                log = Path(self.temporary.name) / "cleanup.log"
+                log.unlink(missing_ok=True)
+                script = CLEANUP + r'''
+flock() { printf 'unlock:%s\n' "$2" >> "$EC_TEST_LOG"; }
+systemctl() {
+    if { : >&8; } 2>/dev/null || { : >&9; } 2>/dev/null; then
+        printf 'lock-still-open\n' >> "$EC_TEST_LOG"
+        return 1
+    fi
+    printf '%s\n' "$*" >> "$EC_TEST_LOG"
+}
+temporary_dir="$EC_TEST_TEMP"
+updater_timer_was_active="$EC_TEST_PREVIOUS"
+installer_finished="$EC_TEST_FINISHED"
+update_lock_held=true
+activity_lock_held=true
+exec 8>"$EC_TEST_LOCK"
+exec 9>"$EC_TEST_LOCK"
+trap installation_cleanup EXIT
+exit "$EC_TEST_EXIT"
+'''
+                environment = {**os.environ, "EC_TEST_LOG": str(log), "EC_TEST_TEMP": str(scratch),
+                               "EC_TEST_PREVIOUS": str(previous).lower(), "EC_TEST_FINISHED": str(finished).lower(),
+                               "EC_TEST_LOCK": str(Path(self.temporary.name) / "test.lock"), "EC_TEST_EXIT": str(code)}
+                result = subprocess.run(["bash", "-c", script], env=environment, capture_output=True)
+                self.assertEqual(result.returncode, code, result.stderr)
+                expected = ["unlock:8", "unlock:9"]
+                if previous and not finished:
+                    expected.append("start electric-cherry-update.timer")
+                self.assertEqual(log.read_text().splitlines(), expected)
+                self.assertFalse(scratch.exists())
+
+
 if __name__ == "__main__":
     unittest.main()

agent/tests/test_uninstall.py

@@ -57,6 +57,9 @@ class UninstallTests(unittest.TestCase):
             "UNINSTALL_EXECUTABLE": self.bin_dir / "electric-cherry-uninstall.py",
             "UNINSTALL_UNIT": self.unit_dir / "electric-cherry-uninstall.service",
             "UNINSTALL_REQUEST": self.config_dir / "decommission.json",
+            "UPDATE_EXECUTABLE": self.bin_dir / "electric-cherry-update.py",
+            "UPDATE_UNIT": self.unit_dir / "electric-cherry-update.service",
+            "UPDATE_TIMER": self.unit_dir / "electric-cherry-update.timer",
         }
         self.patches = [patch.object(module, key, value) for key, value in replacements.items()]
         # Unit tests run without root privileges. Keep actual path, symlink,
@@ -74,8 +77,10 @@ class UninstallTests(unittest.TestCase):
         self.config = {"hostId": "host-one", "controller": "https://controller.example", "credential": "secret", "root": str(self.root), "configPath": str(module.AGENT_CONFIG)}
         self.agent = module.Agent(self.config)
         self.docker = CleanupDocker()
-        for path in (module.AGENT_EXECUTABLE, module.AGENT_UNIT, module.UNINSTALL_EXECUTABLE, module.UNINSTALL_UNIT):
+        for path in (module.AGENT_EXECUTABLE, module.AGENT_UNIT, module.UNINSTALL_EXECUTABLE, module.UNINSTALL_UNIT, module.UPDATE_EXECUTABLE, module.UPDATE_UNIT, module.UPDATE_TIMER):
             path.write_text("installed")
+        for path in (module.AGENT_EXECUTABLE, module.UNINSTALL_EXECUTABLE):
+            path.with_suffix(".py.update").write_text("interrupted update")
         module.atomic_json(module.AGENT_CONFIG, self.config)
         module.atomic_json(self.root / module.ROOT_MARKER, {"version": 1, "root": str(self.root), "hostId": "host-one", "controller": "https://controller.example"})
         self.game = self.root / "libraries" / "t6" / "library" / "content" / "zone" / "all" / "dlc.ff"
@@ -108,6 +113,11 @@ class UninstallTests(unittest.TestCase):
             self.assertFalse(module.AGENT_CONFIG.exists())
             self.assertFalse(module.AGENT_EXECUTABLE.exists())
             self.assertFalse(module.AGENT_UNIT.exists())
+            self.assertFalse(module.UPDATE_EXECUTABLE.exists())
+            self.assertFalse(module.UPDATE_UNIT.exists())
+            self.assertFalse(module.UPDATE_TIMER.exists())
+            self.assertFalse(module.AGENT_EXECUTABLE.with_suffix(".py.update").exists())
+            self.assertFalse(module.UNINSTALL_EXECUTABLE.with_suffix(".py.update").exists())
             if stage == "cleaned":
                 self.assertTrue(module.UNINSTALL_REQUEST.exists())
                 self.assertTrue(module.UNINSTALL_EXECUTABLE.exists())
@@ -120,6 +130,8 @@ class UninstallTests(unittest.TestCase):
         self.assertEqual(stages, ["cleaned", "completed"])
         self.assertEqual((self.base / "unrelated.txt").read_text(), "keep")
         module.systemctl.assert_any_call("disable", "--now", "electric-cherry-agent.service")
+        module.systemctl.assert_any_call("disable", "--now", "electric-cherry-update.timer")
+        module.systemctl.assert_any_call("stop", "electric-cherry-update.service")
         module.systemctl.assert_any_call("disable", "electric-cherry-uninstall.service")
 
     def test_handoff_is_durable_and_never_reports_operation_completed(self):

agent/tests/test_update_integration.py

@@ -0,0 +1,137 @@
+import fcntl
+import importlib.util
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("ec_update_integration", Path(__file__).resolve().parents[1] / "ec_agent.py")
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+
+class UpdateIntegrationTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.base = Path(self.temporary.name)
+        self.config_dir = self.base / "config"
+        self.config_dir.mkdir()
+        replacements = {
+            "AGENT_CONFIG": self.config_dir / "agent.json",
+            "UPDATE_JOURNAL": self.config_dir / "update" / "journal.json",
+            "UPDATE_STATUS": self.config_dir / "update-status.json",
+            "AGENT_HEALTH": self.config_dir / "agent-health.json",
+        }
+        self.patches = [patch.object(module, key, value) for key, value in replacements.items()]
+        self.patches.append(patch.object(module, "protected_file", side_effect=lambda path, **_: path))
+        for entry in self.patches:
+            entry.start()
+        self.agent = module.Agent({"root": str(self.base / "data"), "configPath": str(module.AGENT_CONFIG), "controller": "https://controller.example", "hostId": "one", "credential": "secret"})
+        self.agent.can_update = lambda: True
+        self.agent.inventory = lambda: {"agentVersion": module.VERSION}
+
+    def tearDown(self):
+        self.agent.journal.close()
+        for entry in reversed(self.patches):
+            entry.stop()
+        self.temporary.cleanup()
+
+    def assert_activity_held(self):
+        with (self.agent.root / "agent" / "activity.lock").open("a") as stream:
+            with self.assertRaises(BlockingIOError):
+                fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
+
+    def test_operation_delivery_execution_and_reports_exclude_updates(self):
+        operation = {"id": "one", "kind": "host.inspect", "payload": {}}
+        calls = []
+        def request(method, path, body=None):
+            self.assert_activity_held()
+            calls.append((method, path))
+            return {"operations": [operation]} if method == "GET" else {}
+        def dispatch(value):
+            self.assert_activity_held()
+            return {"ok": True}
+        self.agent.request = request
+        self.agent.dispatch = dispatch
+        self.agent.poll_operations()
+        self.assertEqual(self.agent.journal.get("one")["state"], "completed")
+        self.assertEqual(len(calls), 4)
+        self.assertIsNone(self.agent.current_operation)
+        with self.agent.activity() as acquired:
+            self.assertTrue(acquired)
+
+    def test_updater_lock_defers_delivery_without_blocking_health(self):
+        self.agent.request = lambda method, path, body=None: {"host": {}}
+        with self.agent.activity() as acquired:
+            self.assertTrue(acquired)
+            with patch.object(self.agent, "request", wraps=self.agent.request) as request:
+                self.agent.poll_operations()
+                request.assert_not_called()
+                self.agent.heartbeat()
+                self.assertTrue(module.AGENT_HEALTH.exists())
+
+    def test_journal_blocks_operations_during_verification_and_recovery(self):
+        module.UPDATE_JOURNAL.parent.mkdir()
+        for content in ('{"id":"candidate","phase":"verifying"}', '{"id":"candidate","phase":"rollback"}', '{broken'):
+            module.UPDATE_JOURNAL.write_text(content)
+            with patch.object(self.agent, "request") as request:
+                self.agent.poll_operations()
+                request.assert_not_called()
+        module.UPDATE_JOURNAL.unlink()
+        with patch.object(self.agent, "request", return_value={"operations": []}) as request:
+            self.agent.poll_operations()
+            request.assert_called_once_with("GET", "/api/agent/operations")
+
+    def test_failed_controller_contact_never_writes_healthy_receipt(self):
+        with patch.object(self.agent, "request", side_effect=module.AgentError("offline")):
+            with self.assertRaises(module.AgentError):
+                self.agent.heartbeat()
+        self.assertFalse(module.AGENT_HEALTH.exists())
+
+    def test_health_proves_current_process_version_and_transaction(self):
+        module.atomic_json(module.UPDATE_JOURNAL, {"id": "candidate-id", "phase": "verifying"})
+        with patch.object(self.agent, "request", return_value={"host": {}}), patch.object(module.time, "time", return_value=12345.5):
+            self.agent.heartbeat()
+        self.assertEqual(json.loads(module.AGENT_HEALTH.read_text()), {"version": module.VERSION, "pid": os.getpid(), "checkedAt": 12345.5, "updateId": "candidate-id"})
+        module.UPDATE_JOURNAL.unlink()
+        self.agent.record_health()
+        self.assertNotIn("updateId", json.loads(module.AGENT_HEALTH.read_text()))
+
+    def test_real_agent_health_receipt_is_accepted_by_the_independent_updater(self):
+        updater_spec = importlib.util.spec_from_file_location("ec_update_contract", Path(__file__).resolve().parents[1] / "ec_agent_update.py")
+        updater_module = importlib.util.module_from_spec(updater_spec)
+        updater_spec.loader.exec_module(updater_module)
+        journal = {"id": "candidate-id", "phase": "verifying", "targetVersion": module.VERSION, "switchedAt": module.time.time() - 1}
+        module.atomic_json(module.UPDATE_JOURNAL, journal)
+        with patch.object(self.agent, "request", return_value={"host": {}}):
+            self.agent.heartbeat()
+        updater = updater_module.Updater()
+        with patch.object(updater, "check_uninstall"), patch.object(updater_module, "AGENT_HEALTH", module.AGENT_HEALTH), patch.object(updater_module, "trusted", side_effect=lambda path, **_: path), patch.object(updater_module, "service_state", return_value=(True, os.getpid())):
+            self.assertTrue(updater.wait_healthy(journal))
+
+    def test_operation_exception_releases_update_lock(self):
+        with patch.object(self.agent, "request", side_effect=module.AgentError("offline")):
+            with self.assertRaises(module.AgentError):
+                self.agent.poll_operations()
+        with self.agent.activity() as acquired:
+            self.assertTrue(acquired)
+
+    def test_update_inventory_excludes_local_recovery_and_secret_fields(self):
+        module.atomic_json(module.UPDATE_STATUS, {"state": "failed", "targetVersion": "1.3.0", "currentVersion": "untrusted", "error": "Restored previous agent", "checkedAt": 1700000000, "credential": "secret", "journal": {"private": True}, "failedSha256": "digest"})
+        self.assertEqual(self.agent.update_inventory(), {"enabled": True, "state": "failed", "currentVersion": module.VERSION, "targetVersion": "1.3.0", "error": "Restored previous agent", "checkedAt": "2023-11-14T22:13:20Z"})
+
+    def test_malformed_update_status_is_visible_without_breaking_inventory(self):
+        module.UPDATE_STATUS.write_text("[]")
+        self.assertEqual(self.agent.update_inventory()["state"], "failed")
+
+    def test_shutdown_never_polls_work(self):
+        self.agent.stopping.set()
+        with patch.object(self.agent, "request") as request:
+            self.agent.poll_operations()
+            request.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()

agent/tests/test_updates.py

@@ -0,0 +1,418 @@
+import contextlib
+import fcntl
+import importlib.util
+import io
+import json
+import os
+from pathlib import Path
+import stat
+import tempfile
+import time
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("ec_agent_update", Path(__file__).resolve().parents[1] / "ec_agent_update.py")
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+PREVIOUS = b'VERSION = "1.0.0"\n# previous agent\n'
+CANDIDATE = b'VERSION = "1.1.0"\n# next agent\n'
+
+
+class PowerLoss(BaseException):
+    pass
+
+
+class Response(io.BytesIO):
+    def __init__(self, value, headers=None):
+        super().__init__(value)
+        self.headers = headers or {}
+        self.status = 200
+
+
+class UpdateTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.base = Path(self.temp.name).resolve()
+        self.config_dir = self.base / "etc" / "electric-cherry"
+        self.bin_dir = self.base / "libexec"
+        self.root = self.base / "data"
+        for directory in (self.config_dir, self.bin_dir, self.root / "agent"):
+            directory.mkdir(parents=True)
+        paths = {
+            "AGENT_CONFIG": self.config_dir / "agent.json",
+            "AGENT_EXECUTABLE": self.bin_dir / "electric-cherry-agent.py",
+            "UNINSTALL_EXECUTABLE": self.bin_dir / "electric-cherry-uninstall.py",
+            "UNINSTALL_REQUEST": self.config_dir / "decommission.json",
+            "UPDATE_LOCK": self.config_dir / "update.lock",
+            "UPDATE_DIR": self.config_dir / "update",
+            "UPDATE_JOURNAL": self.config_dir / "update" / "journal.json",
+            "UPDATE_STATUS": self.config_dir / "update-status.json",
+            "AGENT_HEALTH": self.config_dir / "agent-health.json",
+        }
+        self.patches = [patch.object(module, name, value) for name, value in paths.items()]
+        def trusted_for_test(value, missing=False, directory=False):
+            path = Path(value)
+            if any(item.is_symlink() for item in (path, *path.parents)):
+                raise module.UpdateError("Unsafe symbolic link")
+            if not path.exists():
+                if missing:
+                    return path
+                raise module.UpdateError("File unavailable")
+            if directory and not path.is_dir() or not directory and not path.is_file():
+                raise module.UpdateError("Wrong file type")
+            return path
+        self.patches.extend([patch.object(module, "trusted", side_effect=trusted_for_test), patch.object(module, "sync_parent"), patch.object(module, "systemctl", side_effect=self.systemctl)])
+        for item in self.patches:
+            item.start()
+        self.config = {"hostId": "host-one", "root": str(self.root), "controller": "https://controller.example", "credential": "secret-enrollment-credential"}
+        self.config_bytes = json.dumps(self.config).encode()
+        module.AGENT_CONFIG.write_bytes(self.config_bytes)
+        (self.root / ".electric-cherry.json").write_text(json.dumps({"version": 1, "root": str(self.root), "hostId": "host-one", "controller": "https://controller.example"}))
+        module.AGENT_EXECUTABLE.write_bytes(PREVIOUS)
+        module.UNINSTALL_EXECUTABLE.write_bytes(PREVIOUS)
+        (self.root / "games.ff").write_bytes(b"large game files unchanged")
+        self.commands = []
+        self.active = True
+        self.pid = 123
+        self.write_health = True
+        self.downloads = []
+        self.source = CANDIDATE
+        self.release = {"version": "1.1.0", "sha256": module.sha256(CANDIDATE), "protocolVersion": 1, "updaterProtocolVersion": 1, "platform": "debian-amd64"}
+
+    def tearDown(self):
+        for item in reversed(self.patches):
+            item.stop()
+        self.temp.cleanup()
+
+    def systemctl(self, *args):
+        self.commands.append(args)
+        if args == ("stop", module.AGENT_SERVICE):
+            self.active = False
+            return ""
+        if args == ("start", module.AGENT_SERVICE):
+            # The candidate must be able to take the operation lock at startup.
+            with module.exclusive_lock(self.root / "agent" / "activity.lock") as activity:
+                self.assertIsNotNone(activity, "Updater held the operation lock while starting the agent")
+            self.active = True
+            self.pid += 1
+            if self.write_health and module.UPDATE_JOURNAL.exists():
+                journal = json.loads(module.UPDATE_JOURNAL.read_text())
+                module.AGENT_HEALTH.write_text(json.dumps({"version": module.source_version(module.AGENT_EXECUTABLE.read_bytes()), "pid": self.pid, "checkedAt": time.time(), "updateId": journal["id"]}))
+            return ""
+        if args == ("show", module.AGENT_SERVICE, "--property=ActiveState", "--property=MainPID"):
+            return "ActiveState=" + ("active" if self.active else "inactive") + "\nMainPID=" + str(self.pid if self.active else 0) + "\n"
+        self.fail("Unexpected service command " + str(args))
+
+    def fetch(self, path, limit):
+        self.downloads.append(path)
+        if path.endswith("manifest.json"):
+            return json.dumps(self.release).encode()
+        if path.endswith("ec-agent.py"):
+            return self.source
+        self.fail("Unexpected download " + path)
+
+    def updater(self):
+        updater = module.Updater()
+        updater.fetch = self.fetch
+        return updater
+
+    def status(self):
+        return json.loads(module.UPDATE_STATUS.read_text())
+
+    def assert_unchanged(self):
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), PREVIOUS)
+        self.assertEqual(module.UNINSTALL_EXECUTABLE.read_bytes(), PREVIOUS)
+        self.assertEqual(module.AGENT_CONFIG.read_bytes(), self.config_bytes)
+        self.assertEqual((self.root / "games.ff").read_bytes(), b"large game files unchanged")
+
+    def test_verified_upgrade_waits_for_authenticated_candidate_health(self):
+        self.assertEqual(self.updater().run(), "updated")
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(module.UNINSTALL_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(module.AGENT_CONFIG.read_bytes(), self.config_bytes)
+        self.assertEqual((self.root / "games.ff").read_bytes(), b"large game files unchanged")
+        self.assertEqual(self.status()["currentVersion"], "1.1.0")
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+        self.assertFalse((module.UPDATE_DIR / "agent.backup.py").exists())
+        self.assertEqual(next(command for command in self.commands if command[0] != "show"), ("stop", module.AGENT_SERVICE))
+
+    def test_equal_or_older_versions_do_not_download_or_restart(self):
+        for target in ("1.0.0", "0.9.9"):
+            with self.subTest(target=target):
+                self.release["version"] = target
+                self.assertEqual(self.updater().run(), "current")
+                self.assert_unchanged()
+        self.assertFalse(any(path.endswith("ec-agent.py") for path in self.downloads))
+        self.assertEqual(self.commands, [])
+
+    def test_busy_operation_defers_update_without_stopping_agent(self):
+        with module.exclusive_lock(self.root / "agent" / "activity.lock"):
+            self.assertEqual(self.updater().run(), "deferred")
+        self.assertIn("executing an operation", self.status()["error"])
+        self.assert_unchanged()
+        self.assertEqual(self.commands, [])
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+
+    def test_deliberately_stopped_agent_is_not_updated_or_started(self):
+        self.active = False
+        self.assertEqual(self.updater().run(), "deferred")
+        self.assertIn("Agent service is not running", self.status()["error"])
+        self.assert_unchanged()
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+        self.assertFalse(self.active)
+        self.assertFalse(any(command[0] in ("start", "stop") for command in self.commands))
+
+    def test_concurrent_worker_does_not_overwrite_active_worker_status(self):
+        module.UPDATE_STATUS.write_text('{"state":"updating"}')
+        with module.exclusive_lock(module.UPDATE_LOCK):
+            self.assertEqual(self.updater().run(), "deferred")
+        self.assertEqual(self.status(), {"state": "updating"})
+        self.assertEqual(self.downloads, [])
+
+    def test_uninstall_request_prevents_any_download_or_mutation(self):
+        module.UNINSTALL_REQUEST.write_text("{}")
+        self.assertEqual(self.updater().run(), "deferred")
+        self.assertIn("deletion", self.status()["error"])
+        self.assertEqual(self.downloads, [])
+        self.assertEqual(self.commands, [])
+        self.assert_unchanged()
+
+    def test_uninstall_starting_during_download_wins_before_code_changes(self):
+        updater = self.updater()
+        def fetch(path, limit):
+            data = self.fetch(path, limit)
+            if path.endswith("ec-agent.py"):
+                module.UNINSTALL_REQUEST.write_text("{}")
+            return data
+        updater.fetch = fetch
+        self.assertEqual(updater.run(), "deferred")
+        self.assert_unchanged()
+        self.assertEqual(self.commands, [])
+
+    def test_controller_unavailable_keeps_running_agent(self):
+        updater = self.updater()
+        updater.fetch = lambda *_: (_ for _ in ()).throw(module.ContactError("Controller unavailable"))
+        self.assertEqual(updater.run(), "deferred")
+        self.assert_unchanged()
+        self.assertTrue(self.active)
+        self.assertEqual(self.commands, [])
+
+    def test_checksum_failure_holds_artifact_without_executing_it(self):
+        self.source = CANDIDATE + b"# changed in transit"
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertIn("checksum", self.status()["error"])
+        self.assert_unchanged()
+        self.assertEqual(self.commands, [])
+        self.downloads.clear()
+        self.source = CANDIDATE
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertEqual(self.downloads, ["/api/agent/release/manifest.json"])
+
+    def test_invalid_or_mismatched_python_version_is_never_installed(self):
+        for source in (b'VERSION = "1.1.0"\ninvalid syntax ?\n', b'VERSION = "5.0.0"\n'):
+            with self.subTest(source=source):
+                self.source = source
+                self.release["sha256"] = module.sha256(source)
+                self.assertEqual(self.updater().run(), "failed")
+                self.assert_unchanged()
+                self.assertEqual(self.commands, [])
+
+    def test_incompatible_protocol_platform_or_version_never_downloads_code(self):
+        original = dict(self.release)
+        for key, value in (("protocolVersion", 2), ("protocolVersion", True), ("platform", "linux-arm64"), ("version", "1.1.0-rc1"), ("version", "01.1.0"), ("updaterProtocolVersion", 2)):
+            with self.subTest(key=key, value=value):
+                self.release = {**original, key: value}
+                self.assertIn(self.updater().run(), ("failed", "deferred"))
+                self.assert_unchanged()
+        self.assertFalse(any(path.endswith("ec-agent.py") for path in self.downloads))
+        self.assertEqual(self.commands, [])
+
+    def test_failed_health_rolls_back_both_files_then_holds_release(self):
+        updater = self.updater()
+        updater.wait_healthy = lambda journal: journal["targetVersion"] == "1.0.0"
+        self.assertEqual(updater.run(), "failed")
+        self.assert_unchanged()
+        self.assertTrue(self.active)
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+        self.assertEqual(self.status()["currentVersion"], "1.0.0")
+        self.assertEqual(sum(command == ("start", module.AGENT_SERVICE) for command in self.commands), 2)
+        self.downloads.clear()
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertEqual(self.downloads, ["/api/agent/release/manifest.json"])
+
+    def test_changed_artifact_of_failed_target_can_be_retried(self):
+        updater = self.updater()
+        updater.wait_healthy = lambda journal: journal["targetVersion"] == "1.0.0"
+        self.assertEqual(updater.run(), "failed")
+        self.source = CANDIDATE + b"# corrected release\n"
+        self.release["sha256"] = module.sha256(self.source)
+        self.assertEqual(self.updater().run(), "updated")
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), self.source)
+
+    def crash_during_second_replacement(self):
+        original = module.atomic_bytes
+        def interrupt(path, value, mode=0o600):
+            if Path(path) == module.UNINSTALL_EXECUTABLE and value == CANDIDATE:
+                raise PowerLoss()
+            return original(path, value, mode)
+        with patch.object(module, "atomic_bytes", side_effect=interrupt):
+            with self.assertRaises(PowerLoss):
+                self.updater().run()
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(module.UNINSTALL_EXECUTABLE.read_bytes(), PREVIOUS)
+        self.assertTrue(module.UPDATE_JOURNAL.exists())
+
+    def test_crash_between_executable_replacements_recovers_from_backups(self):
+        self.crash_during_second_replacement()
+        self.downloads.clear()
+        self.assertEqual(self.updater().run(), "failed")
+        self.assert_unchanged()
+        self.assertEqual(self.downloads, [])
+        self.assertTrue(self.active)
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+
+    def test_crash_after_verification_still_recovers_conservatively(self):
+        updater = self.updater()
+        updater.finish_journal = lambda: (_ for _ in ()).throw(PowerLoss())
+        with self.assertRaises(PowerLoss):
+            updater.run()
+        self.assertEqual(json.loads(module.UPDATE_JOURNAL.read_text())["phase"], "verifying")
+        self.assertEqual(self.updater().run(), "failed")
+        self.assert_unchanged()
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+
+    def test_corrupt_backup_keeps_journal_and_never_overwrites_code(self):
+        self.crash_during_second_replacement()
+        (module.UPDATE_DIR / "agent.backup.py").write_bytes(b"corrupt")
+        before = len(self.commands)
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertTrue(module.UPDATE_JOURNAL.exists())
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(len(self.commands), before)
+
+    def test_changed_enrollment_prevents_update_and_rollback(self):
+        self.crash_during_second_replacement()
+        changed = {**self.config, "hostId": "different-host"}
+        module.AGENT_CONFIG.write_text(json.dumps(changed))
+        (self.root / ".electric-cherry.json").write_text(json.dumps({"version": 1, "root": str(self.root), "hostId": "different-host", "controller": "https://controller.example"}))
+        before = len(self.commands)
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertTrue(module.UPDATE_JOURNAL.exists())
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+        self.assertEqual(len(self.commands), before)
+
+    def test_rollback_busy_lock_keeps_recovery_journal_for_next_tick(self):
+        self.crash_during_second_replacement()
+        with module.exclusive_lock(self.root / "agent" / "activity.lock"):
+            self.assertEqual(self.updater().run(), "deferred")
+        self.assertEqual(json.loads(module.UPDATE_JOURNAL.read_text())["phase"], "rollback")
+        self.assertEqual(self.updater().run(), "failed")
+        self.assert_unchanged()
+
+    def test_unconfirmed_rollback_health_retains_backups_and_journal(self):
+        updater = self.updater()
+        updater.wait_healthy = lambda _: False
+        self.assertEqual(updater.run(), "failed")
+        self.assert_unchanged()
+        self.assertEqual(json.loads(module.UPDATE_JOURNAL.read_text())["phase"], "rollback")
+        self.assertTrue((module.UPDATE_DIR / "agent.backup.py").exists())
+        self.assertIn("recovery is pending", self.status()["error"])
+        # A subsequent worker can safely confirm the restored release.
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertFalse(module.UPDATE_JOURNAL.exists())
+
+    def test_stale_wrong_pid_wrong_release_or_wrong_update_health_is_rejected(self):
+        updater = self.updater()
+        journal = {"id": "this-update", "targetVersion": "1.1.0", "switchedAt": time.time()}
+        valid = {"version": "1.1.0", "pid": self.pid, "checkedAt": time.time(), "updateId": "this-update"}
+        for key, value in (("pid", self.pid + 1), ("version", "1.0.0"), ("updateId", "previous-update"), ("checkedAt", journal["switchedAt"] - 1), ("checkedAt", float("nan")), ("checkedAt", time.time() + 100)):
+            with self.subTest(key=key, value=value):
+                module.AGENT_HEALTH.write_text(json.dumps({**valid, key: value}))
+                with patch.object(module, "HEALTH_TIMEOUT", 1), patch.object(module.time, "monotonic", side_effect=[0, 0, 2]), patch.object(module.time, "sleep"):
+                    self.assertFalse(updater.wait_healthy(journal))
+        module.AGENT_HEALTH.write_text(json.dumps(valid))
+        self.assertTrue(updater.wait_healthy(journal))
+
+    def test_symlinked_executable_is_rejected_without_touching_target(self):
+        external = self.base / "external.py"
+        external.write_bytes(PREVIOUS)
+        module.AGENT_EXECUTABLE.unlink()
+        module.AGENT_EXECUTABLE.symlink_to(external)
+        self.assertEqual(self.updater().run(), "failed")
+        self.assertEqual(external.read_bytes(), PREVIOUS)
+        self.assertEqual(self.commands, [])
+
+    def test_network_reader_enforces_header_and_stream_bounds(self):
+        updater = module.Updater()
+        updater.load_installation()
+        for response in (Response(b"abc", {"Content-Length": "100"}), Response(b"abcdef")):
+            with self.subTest(response=response), patch.object(module.urllib.request, "build_opener") as opener:
+                opener.return_value.open.return_value = response
+                with self.assertRaises(module.IntegrityError):
+                    updater.fetch("/api/agent/release/ec-agent.py", 5)
+
+    def test_trickling_or_truncated_download_defers_without_holding_release(self):
+        updater = module.Updater()
+        updater.load_installation()
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            opener.return_value.open.return_value = Response(b"source")
+            with patch.object(module.time, "monotonic", side_effect=[0, 0, 121]):
+                with self.assertRaisesRegex(module.ContactError, "timed out"):
+                    updater.fetch("/api/agent/release/ec-agent.py", 100)
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            response = Response(b"source")
+            response.read1 = lambda *_: (_ for _ in ()).throw(module.http.client.IncompleteRead(b"partial"))
+            opener.return_value.open.return_value = response
+            with self.assertRaises(module.ContactError):
+                updater.fetch("/api/agent/release/ec-agent.py", 100)
+
+    def test_stale_code_staging_is_recovered_without_leaving_extra_executables(self):
+        temporary = module.AGENT_EXECUTABLE.with_name(module.AGENT_EXECUTABLE.name + ".update")
+        temporary.write_bytes(b"partial previous download")
+        self.assertEqual(self.updater().run(), "updated")
+        self.assertFalse(temporary.exists())
+        self.assertEqual(module.AGENT_EXECUTABLE.read_bytes(), CANDIDATE)
+
+    def test_download_uses_fixed_https_controller_and_no_enrollment_credentials(self):
+        updater = module.Updater()
+        updater.load_installation()
+        with patch.object(module.urllib.request, "build_opener") as opener:
+            opener.return_value.open.return_value = Response(b"source", {"Content-Length": "6"})
+            self.assertEqual(updater.fetch("/api/agent/release/ec-agent.py", 100), b"source")
+            request = opener.return_value.open.call_args.args[0]
+            self.assertEqual(request.full_url, "https://controller.example/api/agent/release/ec-agent.py")
+            self.assertIsNone(request.get_header("Authorization"))
+            redirect = opener.call_args.args[0]
+            with self.assertRaises(module.ContactError):
+                redirect.redirect_request(None, None, 302, "redirect", {}, "https://other.example")
+
+    def test_insecure_or_credential_embedded_controller_is_rejected(self):
+        for controller in ("http://controller.example", "https://token:secret@controller.example"):
+            with self.subTest(controller=controller):
+                module.AGENT_CONFIG.write_text(json.dumps({**self.config, "controller": controller}))
+                self.assertEqual(self.updater().run(), "failed")
+                self.assertEqual(self.downloads, [])
+                self.assertEqual(self.commands, [])
+
+
+class UpdateSecurityTests(unittest.TestCase):
+    def test_untrusted_ownership_writable_paths_and_special_files_are_rejected(self):
+        for mode, uid in ((stat.S_IFREG | 0o666, 0), (stat.S_IFDIR | 0o755, 1000), (stat.S_IFLNK | 0o777, 0), (stat.S_IFIFO | 0o600, 0)):
+            with self.subTest(mode=mode, uid=uid), patch.object(Path, "lstat", return_value=os.stat_result((mode, 1, 1, 1, uid, 0, 0, 0, 0, 0))):
+                with self.assertRaises(module.UpdateError):
+                    module.trusted("/etc/electric-cherry/update/journal.json")
+
+    def test_source_is_parsed_and_compiled_without_execution(self):
+        source = b'VERSION="1.2.3"\nraise RuntimeError("Never execute downloaded code during verification")\n'
+        self.assertEqual(module.source_version(source), "1.2.3")
+
+    def test_numeric_version_order_and_strict_format(self):
+        self.assertGreater(module.version("1.10.0"), module.version("1.9.9"))
+        for value in ("v1.0.0", "1.0", "1.2.03", "1.0.0+local", "1.0.0-rc1", True, None, "1.0.000000000000"):
+            with self.subTest(value=value), self.assertRaises(module.IntegrityError):
+                module.version(value)
+
+
+if __name__ == "__main__":
+    unittest.main()

docs/authentication.md

@@ -85,6 +85,13 @@ endpoint or accept passwords as command arguments.
 Host agents retain enrollment/host-token authentication under `/api/agent/`.
 Installer and release files remain public so a new machine can enroll. A browser
 login never substitutes for a host credential, and host tokens cannot manage EC.
+The agent and standalone updater source contain no credentials. Automatic
+updates fetch the controller's public HTTPS release manifest and verify its
+checksums and compatibility. The installed host retains its own credential;
+the updater accepts an upgrade only after a fresh authenticated heartbeat from
+the new agent process. Updating agent code does not change browser sessions or
+user roles. See [automatic agent updates](native-hosts.md#automatic-agent-updates)
+for the one-time installer upgrade required by existing hosts.
 Host deletion uses a separate operation-scoped receipt token so the uninstall
 worker can confirm removal and retry its acknowledgment after host credentials
 are revoked. That token authorizes only the corresponding cleanup receipt.

docs/native-hosts.md

@@ -15,8 +15,9 @@ the agent owns local execution and its durable operation journal.
 2. Open **Hosts → Add Host**, choose a name, local game-port range and bind address,
    and generate an enrollment. Run the displayed command on Debian 12/13 amd64.
 3. The installer validates the platform, installs Python 3 and an absent Docker
-   installation, verifies the downloaded agent SHA-256 over HTTPS, enrolls it and
-   enables `electric-cherry-agent.service`. It needs no Git, Node.js or local build.
+   installation, verifies the downloaded agent and updater SHA-256 over HTTPS,
+   enrolls it and enables `electric-cherry-agent.service` and its automatic update
+   timer. It needs no Git, Node.js or local build.
 4. In **Hosts**, prepare a Plutainer runtime image. In **Game Files**, select T5 or
    T6 and upload your local game folder. Name the library and version and publish
    it after the upload finishes. Include the DLC files in that folder; there is
@@ -48,6 +49,59 @@ The agent uses Plutainer's documented [game-file and app mounts](https://github.
 Its log reader supports relative log links that resolve inside the instance app
 directory, while file writes reject symlinks and paths outside that directory.
 
+## Automatic agent updates
+
+New installations enable automatic agent updates by default. Each host checks
+its controller's release over HTTPS about every five minutes, beginning about
+two minutes after boot, with up to 30 seconds of jitter. Updating the controller
+therefore makes its compatible, newer agent release available to enrolled hosts.
+There is no controller command to schedule and no update button to press.
+
+**Existing hosts need the current installer once.** From the host's **Agent
+access**, generate an installer command and rerun it on the same machine, with
+the same controller and data directory. Do not use `--reenroll` for this upgrade.
+The installer preserves host identity, credentials, game files and running
+containers. Agent version 1.2.0 includes automatic update support; the host page
+shows **Setup required** until a fresh heartbeat confirms the updater is installed.
+
+The **Automatic agent updates** panel reports the running version, target
+version, last check, progress and any error. Offline hosts show their last
+reported status. Deferred updates report why they are waiting: for example an
+active agent operation, an unavailable controller or a newer updater bootstrap.
+Checks also defer while the agent service is stopped; an update does not start a
+service that was stopped outside the update workflow.
+The independent updater holds the same execution lock, verifies the release
+checksum, version and supported protocol, replaces agent code, and restarts the
+agent service. It never restarts Docker or game containers. It confirms an
+authenticated heartbeat from the new agent process before accepting the update;
+failure restores the previous agent code. Its durable journal permits recovery
+after a host reboot or interrupted update.
+
+The updater only installs newer compatible versions; rolling the controller back
+does not downgrade agents. A rejected version/checksum pair stays blocked until
+the controller publishes a different pair that is still newer than the installed
+agent. Inspect the reported error when a host
+shows **Needs attention**. The updater itself is a stable bootstrap component:
+it changes only when the installer is rerun. If a future release requires an
+unsupported updater protocol, rerun that release's installer on the host.
+
+When publishing changed agent code, bump its numeric `VERSION` in
+`agent/ec_agent.py` and rebuild the controller. Packaging derives the manifest
+version and checksums from the sources; an already installed version is never
+replaced just because its bytes changed.
+
+The installer places the updater at
+`/usr/local/libexec/electric-cherry-update.py` and enables
+`electric-cherry-update.timer`, which starts the oneshot
+`electric-cherry-update.service`. Its state is kept under `/etc/electric-cherry`,
+including `update-status.json`, `update/journal.json`, `agent-health.json` and
+the shared `update.lock`. Check the timer and worker logs on the host with:
+
+```sh
+systemctl status electric-cherry-update.timer
+journalctl -u electric-cherry-update.service -n 100
+```
+
 ## Identity, transport and credentials
 
 Enrollment tokens are random, hashed at rest, single-use and expire after 15
@@ -81,6 +135,10 @@ API outline:
   returns `202` while remote removal is pending, `200` for an unenrolled host,
   or `409` when the agent needs updating or unresolved ownership remains.
 - `POST /api/agent/enroll`: consume enrollment token and return host credentials.
+- `GET /api/agent/release/manifest.json`: public agent version, protocol and
+  checksum plus `updaterProtocolVersion` and `updaterSha256` for installer bootstrap.
+- `GET /api/agent/release/ec-agent.py` and `ec-agent-update.py`: public release
+  source, verified against that HTTPS manifest before installation.
 - `POST /api/agent/heartbeat`: authenticated inventory, current operation and caches.
 - `GET /api/agent/operations`: deliver/reconcile durable operation IDs.
 - `POST /api/agent/operations/:id`: authenticated acceptance, progress or result.
@@ -142,14 +200,15 @@ Drain the host or remove its servers, then let outstanding operations reconcile.
 An unenrolled host is removed immediately. An enrolled host stays visible as
 **Deleting** until its agent confirms cleanup. Offline hosts wait for their next
 connection; their absence is never treated as proof of removal. Existing agents
-need the 1.1.0 installer: generate a fresh command under **Agent access**, rerun it
+need the current installer: generate a fresh command under **Agent access**, rerun it
 on the host, and wait for a heartbeat advertising uninstall support.
 
 Deletion blocks new operations, cancels queued commands that were never delivered,
 and invalidates unused installer links. A separate cleanup service removes the
 host's downloaded game libraries, instance files, agent journal and configuration,
 owned containers, and recorded runtime images unused by other containers. It
-stops and removes the agent service and executable, then removes its own files.
+stops and removes the agent service and executable, automatic update timer,
+update service, updater executable and update state, then removes its own files.
 The controller hides the host and revokes its ordinary credentials only after
 receiving the final uninstall receipt. Adding the machine again requires a new
 host enrollment.
@@ -217,9 +276,11 @@ npm run test:agent
 npm run build
 ```
 
-The build packages `agent/ec_agent.py` and `agent/install.sh` into the static
-release, generates its checksum manifest and serves those same assets in the
-Node-adapter image. The Python agent uses only its standard library.
+The build packages the agent, standalone updater and installer into the static
+release, generates the agent/updater checksum manifest and serves those same
+assets in the Node-adapter image. Development serves the freshly packaged static
+release even when an older production build exists. The Python programs use only
+the standard library.
 
 Automated tests cover enrollment/authentication, rotation/revocation, host
 readiness, scoped ports, immutable library publication, streaming/ranges,
@@ -235,6 +296,12 @@ rollout. This workspace has no Docker engine or Debian game host, so automated
 coverage is not evidence that a real T5/T6 server has been deployed here. Run:
 
 - Installer on Debian 12 and 13 amd64, then rerun without changing identity/data.
+- Bootstrap an existing host with the current installer, then publish a newer
+  compatible agent release. Verify a timer-driven update, its authenticated
+  heartbeat and unchanged game containers. Exercise an active-operation hold,
+  bad checksum, failed startup/heartbeat rollback, controller downgrade, network
+  loss and host reboot during update. Verify updater protocol mismatch requests
+  an installer rerun.
 - T5 and T6 deployments and a mixed collection from a fresh destination host.
 - Start, stop, restart, rebuild, destroy; verify RCON, map and runtime revision.
 - Read console and game logs, including Plutainer's relative game-log links.
@@ -243,7 +310,7 @@ coverage is not evidence that a real T5/T6 server has been deployed here. Run:
   agent holds disruption.
 - Drain occupied lobbies and verify shared libraries survive their retirement.
 - Delete a drained host; verify its game copies, data directory, agent and cleanup
-  services/configuration/executables are removed, controller uploads/history stay,
+  services, update timer, configuration and executables are removed, controller uploads/history stay,
   and unrelated Docker workloads continue. Repeat with connection loss during
   cleanup and after the final receipt, and a reboot during data removal.
 - Corrupt/interrupted library downloads, resumed transfer and verified promotion.

src/lib/hosts.ts

@@ -24,6 +24,40 @@ export function hostDeletionMessage(host: Pick<Host, 'connected' | 'decommission
 }
 
 export interface HostLibrary { id: string; state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; progress?: number; error?: string }
+export interface HostAgentUpdate {
+    enabled: boolean;
+    state: 'current' | 'checking' | 'deferred' | 'updating' | 'updated' | 'failed';
+    currentVersion: string;
+    targetVersion?: string;
+    error?: string;
+    checkedAt?: string;
+}
+
+export function hostAgentUpdatesEnabled(inventory: HostInventory): boolean {
+    return inventory.agentUpdate?.enabled === true && inventory.capabilities?.includes('agent.update') === true;
+}
+
+export function hostAgentUpdateLabel(inventory: HostInventory): string {
+    if (!hostAgentUpdatesEnabled(inventory)) return 'Setup required';
+    const update = inventory.agentUpdate!;
+    if (update.state === 'current' && !update.checkedAt) return 'Awaiting first check';
+    return { current: 'Current', checking: 'Checking', deferred: 'Deferred',
+        updating: 'Updating', updated: 'Updated', failed: 'Needs attention' }[update.state];
+}
+
+export function hostAgentUpdateMessage(inventory: HostInventory): string {
+    if (!hostAgentUpdatesEnabled(inventory)) return 'Run the current installer once on this host to enable automatic agent updates.';
+    const update = inventory.agentUpdate!;
+    switch (update.state) {
+        case 'current': return update.checkedAt ? 'The last check found no newer compatible agent release.' : 'The host timer will check for a compatible agent release automatically.';
+        case 'checking': return 'The host is checking the controller release and verifying its download.';
+        case 'deferred': return 'The updater is waiting for the reported condition to clear. The host checks again automatically.';
+        case 'updating': return 'The host is replacing agent code and checking that the new process connects successfully.';
+        case 'updated': return 'The new agent connected successfully. Automatic checks continue.';
+        case 'failed': return 'The update did not complete successfully. Review the reported error before retrying setup.';
+    }
+}
+
 export interface HostInventory {
     hostname?: string; agentVersion?: string; protocolVersion?: number; os?: string; architecture?: string;
     cpu?: string; cpuCount?: number; memoryTotal?: number; memoryAvailable?: number;
@@ -31,6 +65,7 @@ export interface HostInventory {
     directoriesWritable?: boolean; runtimeImages?: string[]; libraries?: HostLibrary[];
     currentOperation?: string | null; [key: string]: unknown;
     capabilities?: string[];
+    agentUpdate?: HostAgentUpdate;
 }
 export interface Host {
     id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number;

src/lib/server/auth/guard.ts

@@ -5,7 +5,7 @@ import { roleAllows, routePermission, type RoutePermission } from './access.js';
 
 function publicAsset(routeId: string | null, path: string, method: string) {
     return routeId === null && ['GET','HEAD'].includes(method) &&
-        (path.startsWith('/_app/') || path === '/favicon.svg' || ['/agent/ec-agent.py','/agent/install.sh','/agent/manifest.json'].includes(path));
+        (path.startsWith('/_app/') || path === '/favicon.svg' || ['/agent/ec-agent.py','/agent/ec-agent-update.py','/agent/install.sh','/agent/manifest.json'].includes(path));
 }
 
 /** Streaming responses must lose access when the underlying session does. */

src/lib/server/hosts/index.ts

@@ -208,6 +208,22 @@ export function validateInventory(value: unknown): HostInventory {
     for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
     if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
     if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
+    if (inventory.agentUpdate !== undefined) {
+        const update = inventory.agentUpdate;
+        const version = (value: unknown) => typeof value === 'string' && value.length <= 80 && /^\d+\.\d+\.\d+$/.test(value);
+        if (!update || typeof update !== 'object' || Array.isArray(update) || typeof update.enabled !== 'boolean'
+            || !['current','checking','deferred','updating','updated','failed'].includes(update.state)
+            || !version(update.currentVersion) || (update.targetVersion !== undefined && !version(update.targetVersion))
+            || (update.error !== undefined && (typeof update.error !== 'string' || update.error.length > 2000))
+            || (update.checkedAt !== undefined && (typeof update.checkedAt !== 'string' || update.checkedAt.length > 80 || !Number.isFinite(Date.parse(update.checkedAt))))) {
+            throw new Error('Invalid agent update inventory.');
+        }
+        // Only expose display fields, never the local recovery journal or secrets.
+        inventory.agentUpdate = { enabled: update.enabled, state: update.state, currentVersion: update.currentVersion,
+            ...(update.targetVersion === undefined ? {} : { targetVersion: update.targetVersion }),
+            ...(update.error === undefined ? {} : { error: update.error }),
+            ...(update.checkedAt === undefined ? {} : { checkedAt: update.checkedAt }) };
+    }
     if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library =>
         !library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.');
     return inventory;

src/lib/server/hosts/releases.ts

@@ -2,7 +2,9 @@ import { readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 
 /** adapter-node copies public assets to build/client; Vite serves static in development. */
-export async function agentReleaseAsset(file: 'install.sh' | 'ec-agent.py' | 'manifest.json'): Promise<string> {
+export async function agentReleaseAsset(file: 'install.sh' | 'ec-agent.py' | 'ec-agent-update.py' | 'manifest.json'): Promise<string> {
+    // A prior production build must never shadow a freshly packaged development release.
+    if (process.env.NODE_ENV === 'development') return readFile(resolve('static/agent', file), 'utf8');
     try { return await readFile(resolve('build/client/agent', file), 'utf8'); }
     catch (cause) {
         if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause;

src/routes/api/agent/release/[file]/+server.ts

@@ -1,6 +1,6 @@
 import { error } from '@sveltejs/kit';
 import { agentReleaseAsset } from '$lib/server/hosts/releases.js';
 export async function GET({ params }: { params: { file: string } }) {
-    if (!['ec-agent.py','manifest.json'].includes(params.file)) error(404, 'Release asset not found.');
-    return new Response(await agentReleaseAsset(params.file as 'ec-agent.py' | 'manifest.json'), { headers: { 'Content-Type': params.file.endsWith('.json') ? 'application/json' : 'text/plain', 'Cache-Control': 'no-cache' } });
+    if (!['ec-agent.py','ec-agent-update.py','manifest.json'].includes(params.file)) error(404, 'Release asset not found.');
+    return new Response(await agentReleaseAsset(params.file as 'ec-agent.py' | 'ec-agent-update.py' | 'manifest.json'), { headers: { 'Content-Type': params.file.endsWith('.json') ? 'application/json' : 'text/plain', 'Cache-Control': 'no-cache' } });
 }

src/routes/hosts/+page.svelte

@@ -2,7 +2,7 @@
     import { onMount } from 'svelte';
     import { page } from '$app/stores';
     import type { Host } from '$lib/hosts.js';
-    import { hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
+    import { hostAgentUpdateLabel, hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Button from '$ui/components/primitives/Button.svelte';
     import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
@@ -51,7 +51,7 @@
         {#if enrollment}<div class="enrollment" role="status"><h3>Install the agent</h3><p>This single-use command expires {new Date(enrollment.expiresAt).toLocaleString()}. Run it with administrator access on your game host.</p><pre>{enrollment.command}</pre><a href={`/hosts/${enrollment.hostId}`}>Open host setup →</a></div>{/if}
     </section>{:else}<p>Administrators manage host setup, runtime preparation, and credentials.</p>{/if}
     {#if loading}<p role="status">Loading hosts…</p>{:else if !hosts.length}<p>No hosts connected yet.{canManage ? ' Add a host to begin.' : ''}</p>{:else}
-        <div class="host-grid">{#each hosts as host (host.id)}<article class="panel host"><div class="row"><h2><a href={`/hosts/${host.id}`}>{host.name}</a></h2><span class:ready={host.readiness === 'ready'} class="state">{hostReadinessLabel(host.readiness)}</span></div><p>{host.hostname || 'Awaiting enrollment'} · {host.connectionState}</p><p>{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}</p><p>Game ports {host.portMin}–{host.portMax}</p>{#if host.decommission}<p class="reason">{hostDeletionMessage(host)}</p>{#if host.decommission.error}<p class="ui-error">{host.decommission.error}</p>{/if}{:else if host.readinessReasons.length}<p class="reason">{host.readinessReasons[0]}</p>{/if}<div class="host-actions"><Button size="sm" href={`/hosts/${host.id}`}>{host.decommission ? 'View cleanup' : 'Manage host'}</Button>{#if host.readiness === 'needs-game-files'}<Button size="sm" href="/game-files">Choose game files</Button>{/if}{#if canManage && (!host.decommission || host.decommission.retryable)}<Button size="sm" variant="danger" onclick={() => { deletingHost = host; deleteOpen = true; }}>{host.decommission ? 'Retry cleanup' : 'Delete host'}</Button>{/if}</div></article>{/each}</div>
+        <div class="host-grid">{#each hosts as host (host.id)}<article class="panel host"><div class="row"><h2><a href={`/hosts/${host.id}`}>{host.name}</a></h2><span class:ready={host.readiness === 'ready'} class="state">{hostReadinessLabel(host.readiness)}</span></div><p>{host.hostname || 'Awaiting enrollment'} · {host.connectionState}</p><p>{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}</p><p>Game ports {host.portMin}–{host.portMax}</p>{#if host.enrolled}<p>Agent {host.inventory.agentVersion ?? 'unknown'} · Auto updates: {hostAgentUpdateLabel(host.inventory)}</p>{/if}{#if host.decommission}<p class="reason">{hostDeletionMessage(host)}</p>{#if host.decommission.error}<p class="ui-error">{host.decommission.error}</p>{/if}{:else if host.readinessReasons.length}<p class="reason">{host.readinessReasons[0]}</p>{/if}<div class="host-actions"><Button size="sm" href={`/hosts/${host.id}`}>{host.decommission ? 'View cleanup' : 'Manage host'}</Button>{#if host.readiness === 'needs-game-files'}<Button size="sm" href="/game-files">Choose game files</Button>{/if}{#if canManage && (!host.decommission || host.decommission.retryable)}<Button size="sm" variant="danger" onclick={() => { deletingHost = host; deleteOpen = true; }}>{host.decommission ? 'Retry cleanup' : 'Delete host'}</Button>{/if}</div></article>{/each}</div>
     {/if}
 </section>
 <DeleteHostDialog bind:open={deleteOpen} host={deletingHost} ondeleted={deleted} />

src/routes/hosts/[id]/+page.svelte

@@ -3,7 +3,7 @@
     import { page } from '$app/stores';
     import { goto } from '$app/navigation';
     import type { Host, HostOperation, HostMode } from '$lib/hosts.js';
-    import { hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
+    import { hostAgentUpdateLabel, hostAgentUpdateMessage, hostAgentUpdatesEnabled, hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Button from '$ui/components/primitives/Button.svelte';
     import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
@@ -50,6 +50,13 @@
         </dl></section><section class="panel"><h2>Host configuration</h2>{#if canManage}<form onsubmit={save}><fieldset disabled={busy || cleaning}>
             <label>Friendly name<input required maxlength="200" bind:value={name} /></label><div class="fields"><label>First game port<input type="number" min="1" max="65535" required bind:value={portMin} /></label><label>Last game port<input type="number" min={portMin} max="65535" required bind:value={portMax} /></label></div><label>Local bind address<input required bind:value={bindAddress} /></label><label>Public / NAT address<input bind:value={publicAddress} placeholder="Optional public hostname or address" /></label><label>Minimum free game storage (GiB)<input type="number" min="0" step="0.1" required bind:value={minimumFreeGB} /></label><Button type="submit" disabled={busy}>Save settings</Button>
         </fieldset></form>{:else}<dl><dt>Friendly name</dt><dd>{host.name}</dd><dt>Game ports</dt><dd>{host.portMin}–{host.portMax}</dd><dt>Local bind address</dt><dd>{host.bindAddress}</dd><dt>Public / NAT address</dt><dd>{host.publicAddress || 'Not configured'}</dd><dt>Minimum free storage</dt><dd>{bytes(host.minimumFreeBytes)}</dd></dl><p>Host configuration is managed by administrators.</p>{/if}<p>Public addressing is separate from local port reservations. Configure your firewall and NAT for the game-port range.</p></section></div>
+        <section class="panel" aria-live="polite"><h2>Automatic agent updates · {host.enrolled ? hostAgentUpdateLabel(host.inventory) : 'Awaiting installation'}</h2>
+            <dl><dt>Running version</dt><dd>{host.inventory.agentVersion ?? host.inventory.agentUpdate?.currentVersion ?? 'Not reported'}</dd><dt>Target version</dt><dd>{host.inventory.agentUpdate?.targetVersion ?? 'None reported'}</dd><dt>Last update check</dt><dd>{host.inventory.agentUpdate?.checkedAt ? new Date(host.inventory.agentUpdate.checkedAt).toLocaleString() : 'Not reported'}</dd></dl>
+            {#if !host.connected && host.enrolled}<p>This is the last reported update status. The host is offline.</p>{/if}
+            {#if host.inventory.agentUpdate?.error}<p class="ui-error">{host.inventory.agentUpdate.error}</p>{/if}
+            {#if host.enrolled}<p>{hostAgentUpdateMessage(host.inventory)}</p>{:else}<p>The installer enables automatic updates when this host first enrolls.</p>{/if}
+            {#if host.enrolled && !hostAgentUpdatesEnabled(host.inventory)}<p>{canManage ? 'Generate an installer under Agent access below, run it on this host, and wait for a fresh heartbeat. The installer preserves the existing host identity and game data.' : 'An administrator can generate an installer under Agent access and run it on this host.'}</p>{:else if hostAgentUpdatesEnabled(host.inventory)}<p>The host checks this controller about every five minutes. Updates replace agent code between operations; running game containers continue.</p>{/if}
+        </section>
         <div class="grid"><section class="panel"><h2>Game files and runtime</h2><p><a href="/game-files">Manage game-file libraries →</a></p>{#if host.inventory.libraries?.length}<ul>{#each host.inventory.libraries as library}<li>{library.id}: {library.state}{library.progress !== undefined ? ` (${library.progress}%)` : ''}</li>{/each}</ul>{:else}<p>No cached game libraries reported.</p>{/if}
             {#if canManage}<form onsubmit={(event) => { event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}><fieldset disabled={busy || cleaning}><label>Plutainer image<input required bind:value={runtimeImage} /></label><Button type="submit" disabled={busy || !host.connected}>Prepare runtime</Button></fieldset></form>{/if}<p>Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported'}</p>
         </section><section class="panel"><h2>Agent access</h2><p>Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.</p>{#if canManage}<div class="actions"><Button disabled={busy || (cleaning && !host.decommission?.retryable)} onclick={() => action('/enrollment', 'POST')}>Generate installer</Button><Button disabled={busy || cleaning || !host.credentialActive || !host.connected} onclick={() => action('/credentials', 'POST', undefined, 'Credential rotation queued.')}>Rotate credential</Button><Button disabled={busy || cleaning || !host.credentialActive} onclick={() => action('/credentials', 'DELETE', undefined, 'Host credentials revoked. Generate a new installer to enroll again.')}>Revoke credential</Button></div>

static/agent/ec-agent-update.py

@@ -0,0 +1,479 @@
+#!/usr/bin/env python3
+"""Stable Electric Cherry agent updater; only verified agent code is replaced.
+
+The installer owns this bootstrap and its systemd timer. Updates never execute
+downloaded code before installing it, alter game containers, or edit enrollment
+credentials. A durable journal makes an interrupted switch roll back safely.
+"""
+
+import ast
+import contextlib
+import fcntl
+import hashlib
+import http.client
+import json
+import math
+import os
+from pathlib import Path
+import re
+import stat
+import subprocess
+import sys
+import time
+import urllib.parse
+import urllib.request
+import uuid
+
+UPDATER_PROTOCOL_VERSION = 1
+AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
+AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
+UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
+UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
+UPDATE_LOCK = Path("/etc/electric-cherry/update.lock")
+UPDATE_DIR = Path("/etc/electric-cherry/update")
+UPDATE_JOURNAL = UPDATE_DIR / "journal.json"
+UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
+AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
+AGENT_SERVICE = "electric-cherry-agent.service"
+MAX_MANIFEST = 16 * 1024
+MAX_SOURCE = 8 * 1024 * 1024
+HEALTH_TIMEOUT = 90
+VERSION_RE = re.compile(r"^(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})$")
+DIGEST_RE = re.compile(r"^[0-9a-f]{64}$")
+
+
+class UpdateError(Exception):
+    pass
+
+
+class IntegrityError(UpdateError):
+    pass
+
+
+class ContactError(UpdateError):
+    pass
+
+
+class Deferred(UpdateError):
+    pass
+
+
+def trusted(path, missing=False, directory=False):
+    path = Path(path)
+    for current in (*reversed(path.parents), path):
+        try:
+            info = current.lstat()
+        except FileNotFoundError:
+            if missing:
+                continue
+            raise UpdateError("Agent update files are unavailable; repair the installation") from None
+        expected_type = stat.S_ISDIR if current != path or directory else stat.S_ISREG
+        if not expected_type(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+            raise UpdateError("Agent update paths must be regular root-owned files and directories that other users cannot modify")
+    return path
+
+
+def sync_parent(path):
+    descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def atomic_bytes(path, value, mode=0o600):
+    path = trusted(path, missing=True)
+    trusted(path.parent, directory=True)
+    # Code staging uses known names so recovery and host removal can also
+    # remove a file left behind by a power loss during the atomic write.
+    temporary = path.with_name(path.name + ".update") if path in (AGENT_EXECUTABLE, UNINSTALL_EXECUTABLE) else path.with_name(path.name + "." + uuid.uuid4().hex + ".tmp")
+    try:
+        if temporary.exists() or temporary.is_symlink():
+            trusted(temporary)
+            temporary.unlink()
+            sync_parent(temporary)
+        descriptor = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, mode)
+        with os.fdopen(descriptor, "wb") as stream:
+            stream.write(value)
+            stream.flush()
+            os.fsync(stream.fileno())
+        trusted(path, missing=True)
+        os.replace(temporary, path)
+        sync_parent(path)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def atomic_json(path, value):
+    atomic_bytes(path, json.dumps(value, sort_keys=True, allow_nan=False).encode())
+
+
+def read_bytes(path, limit):
+    trusted(path)
+    with open(path, "rb") as stream:
+        value = stream.read(limit + 1)
+    if len(value) > limit:
+        raise UpdateError("An agent update file exceeds its size limit")
+    return value
+
+
+def read_json(path, limit=MAX_MANIFEST):
+    try:
+        value = json.loads(read_bytes(path, limit))
+    except (ValueError, UnicodeError):
+        raise UpdateError("Agent update metadata is invalid") from None
+    if not isinstance(value, dict):
+        raise UpdateError("Agent update metadata is invalid")
+    return value
+
+
+def sha256(value):
+    return hashlib.sha256(value).hexdigest()
+
+
+def version(value):
+    if not isinstance(value, str) or not VERSION_RE.fullmatch(value):
+        raise IntegrityError("Agent releases must use a numeric major.minor.patch version")
+    return tuple(int(part) for part in value.split("."))
+
+
+def source_version(source):
+    try:
+        tree = ast.parse(source, filename="electric-cherry-agent.py")
+        compile(tree, "electric-cherry-agent.py", "exec")
+    except (SyntaxError, ValueError, TypeError, RecursionError, MemoryError):
+        raise IntegrityError("The agent release is not valid Python source") from None
+    declarations = [node.value.value for node in tree.body if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == "VERSION" for target in node.targets) and isinstance(node.value, ast.Constant)]
+    if len(declarations) != 1:
+        raise IntegrityError("The agent release must declare one explicit VERSION")
+    version(declarations[0])
+    return declarations[0]
+
+
+@contextlib.contextmanager
+def exclusive_lock(path):
+    trusted(path, missing=True)
+    trusted(Path(path).parent, directory=True)
+    descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
+    with os.fdopen(descriptor, "r+") as stream:
+        try:
+            fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            yield None
+            return
+        try:
+            yield stream
+        finally:
+            fcntl.flock(stream, fcntl.LOCK_UN)
+
+
+def systemctl(*args):
+    try:
+        result = subprocess.run(["systemctl", *args], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=60, check=False)
+    except (OSError, subprocess.TimeoutExpired):
+        raise UpdateError("Agent service management is unavailable") from None
+    if result.returncode:
+        raise UpdateError("Agent service management did not complete")
+    return result.stdout.decode("utf-8", "replace")
+
+
+def service_state():
+    output = systemctl("show", AGENT_SERVICE, "--property=ActiveState", "--property=MainPID")
+    fields = dict(line.split("=", 1) for line in output.splitlines() if "=" in line)
+    try:
+        return fields.get("ActiveState") == "active", int(fields.get("MainPID", "0"))
+    except ValueError:
+        return False, 0
+
+
+class Updater:
+    def __init__(self):
+        self.installation = None
+        self.current_version = None
+        self.target = None
+
+    def load_installation(self):
+        config = read_json(AGENT_CONFIG, 64 * 1024)
+        controller = config.get("controller")
+        root_value = config.get("root")
+        host_id = config.get("hostId")
+        if not isinstance(controller, str) or not isinstance(root_value, str) or not isinstance(host_id, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,127}", host_id):
+            raise UpdateError("Agent enrollment is incomplete")
+        parsed = urllib.parse.urlsplit(controller)
+        if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise UpdateError("Agent updates require an HTTPS controller without embedded credentials")
+        root = Path(root_value)
+        if not re.fullmatch(r"/[A-Za-z0-9/_.-]+", root_value) or str(root) != root_value or ".." in root.parts or len(root.parts) < 3:
+            raise UpdateError("The enrolled agent data directory is invalid")
+        trusted(root, directory=True)
+        trusted(root / "agent", directory=True)
+        marker = read_json(root / ".electric-cherry.json")
+        controller = controller.rstrip("/")
+        expected = {"version": 1, "root": root_value, "hostId": host_id, "controller": controller}
+        if marker != expected:
+            raise UpdateError("Agent data ownership does not match this enrollment")
+        info = root.stat()
+        installation = {"hostId": host_id, "controller": controller, "root": root_value, "rootIdentity": [info.st_dev, info.st_ino]}
+        if self.installation is not None and installation != self.installation:
+            raise UpdateError("Agent enrollment changed during the update; no code was replaced")
+        self.installation = installation
+        return installation
+
+    def ensure_directory(self):
+        trusted(UPDATE_DIR, missing=True, directory=True)
+        trusted(UPDATE_DIR.parent, directory=True)
+        if not UPDATE_DIR.exists():
+            UPDATE_DIR.mkdir(mode=0o700)
+            sync_parent(UPDATE_DIR)
+
+    def check_uninstall(self):
+        if UNINSTALL_REQUEST.exists() or UNINSTALL_REQUEST.is_symlink():
+            raise Deferred("Host deletion is pending; agent updates are paused")
+
+    def status(self, state, error=None, current=None):
+        value = {"state": state, "checkedAt": time.time()}
+        if current or self.current_version:
+            value["currentVersion"] = current or self.current_version
+        if self.target:
+            value["targetVersion"] = self.target["version"]
+        if error:
+            value["error"] = str(error)[:500]
+        atomic_json(UPDATE_STATUS, value)
+        return state
+
+    def fetch(self, path, limit):
+        request = urllib.request.Request(self.installation["controller"] + path, headers={"User-Agent": "electric-cherry-updater/1"})
+        class NoRedirect(urllib.request.HTTPRedirectHandler):
+            def redirect_request(self, req, fp, code, msg, headers, newurl):
+                raise ContactError("Agent release redirects are not allowed")
+        try:
+            with urllib.request.build_opener(NoRedirect()).open(request, timeout=25) as response:
+                if response.status != 200:
+                    raise ContactError("The controller did not return a complete agent release")
+                length = response.headers.get("Content-Length")
+                if length is not None and (not length.isdecimal() or int(length) > limit):
+                    raise IntegrityError("The controller release exceeds its download size limit")
+                chunks, total = [], 0
+                deadline = time.monotonic() + 120
+                while True:
+                    if time.monotonic() >= deadline:
+                        raise ContactError("The controller release download timed out")
+                    # HTTPResponse.read1 performs one socket read. read(n) can
+                    # wait indefinitely for n bytes from a trickling server.
+                    block = response.read1(min(64 * 1024, limit + 1 - total))
+                    if not block:
+                        break
+                    total += len(block)
+                    if total > limit:
+                        raise IntegrityError("The controller release exceeds its download size limit")
+                    chunks.append(block)
+                if length is not None and total != int(length):
+                    raise ContactError("The controller release download was interrupted")
+                return b"".join(chunks)
+        except (OSError, ValueError, http.client.HTTPException):
+            raise ContactError("The controller release is unavailable; the running agent was left unchanged") from None
+
+    def manifest(self):
+        try:
+            value = json.loads(self.fetch("/api/agent/release/manifest.json", MAX_MANIFEST))
+        except (ValueError, UnicodeError):
+            raise IntegrityError("The controller release manifest is invalid") from None
+        if not isinstance(value, dict) or type(value.get("protocolVersion")) is not int or value["protocolVersion"] != 1 or value.get("platform") != "debian-amd64":
+            raise IntegrityError("The controller release is not compatible with this host agent")
+        if type(value.get("updaterProtocolVersion")) is not int or value["updaterProtocolVersion"] != UPDATER_PROTOCOL_VERSION:
+            raise Deferred("This release needs a newer updater bootstrap; rerun the host installer")
+        version(value.get("version"))
+        if not isinstance(value.get("sha256"), str) or not DIGEST_RE.fullmatch(value["sha256"]):
+            raise IntegrityError("The controller release manifest has an invalid checksum")
+        return {"version": value["version"], "sha256": value["sha256"]}
+
+    def hold(self, target, error):
+        self.ensure_directory()
+        atomic_json(UPDATE_DIR / "failed-release.json", {**target, "error": error, "failedAt": time.time()})
+
+    def release_held(self):
+        path = UPDATE_DIR / "failed-release.json"
+        if not path.exists():
+            return False
+        held = read_json(path)
+        return all(held.get(key) == self.target[key] for key in ("version", "sha256"))
+
+    def save_journal(self, journal, **changes):
+        journal.update(changes)
+        atomic_json(UPDATE_JOURNAL, journal)
+
+    def read_journal(self):
+        journal = read_json(UPDATE_JOURNAL)
+        if journal.get("installation") != self.installation or journal.get("phase") not in ("installing", "verifying", "rollback") or not isinstance(journal.get("id"), str) or not re.fullmatch(r"[a-f0-9]{32}", journal["id"]):
+            raise UpdateError("Pending agent update ownership or recovery metadata is invalid")
+        version(journal.get("previousVersion"))
+        version(journal.get("targetVersion"))
+        for field in ("targetSha256", "agentBackupSha256", "uninstallBackupSha256"):
+            if not isinstance(journal.get(field), str) or not DIGEST_RE.fullmatch(journal[field]):
+                raise UpdateError("Pending agent update recovery checksums are invalid")
+        return journal
+
+    def finish_journal(self):
+        trusted(UPDATE_JOURNAL)
+        UPDATE_JOURNAL.unlink()
+        sync_parent(UPDATE_JOURNAL)
+        # Removing the journal commits the decision. Backups are disposable
+        # only afterward, so a crash never leaves recovery without its inputs.
+        for name in ("agent.backup.py", "uninstall.backup.py", "candidate.py"):
+            path = trusted(UPDATE_DIR / name, missing=True)
+            path.unlink(missing_ok=True)
+        sync_parent(UPDATE_JOURNAL)
+
+    def activity_lock(self):
+        return exclusive_lock(Path(self.installation["root"]) / "agent" / "activity.lock")
+
+    def install(self, source):
+        with self.activity_lock() as activity:
+            if activity is None:
+                raise Deferred("The agent is executing an operation; update will retry when it is idle")
+            self.check_uninstall()
+            self.load_installation()
+            active, pid = service_state()
+            if not active or pid <= 0:
+                raise Deferred("Agent service is not running; automatic updates wait until it is started")
+            previous = read_bytes(AGENT_EXECUTABLE, MAX_SOURCE)
+            uninstall = read_bytes(UNINSTALL_EXECUTABLE, MAX_SOURCE)
+            if source_version(previous) != self.current_version:
+                raise Deferred("The installed agent changed while checking for updates")
+            source_version(uninstall)
+            self.ensure_directory()
+            atomic_bytes(UPDATE_DIR / "agent.backup.py", previous)
+            atomic_bytes(UPDATE_DIR / "uninstall.backup.py", uninstall)
+            atomic_bytes(UPDATE_DIR / "candidate.py", source)
+            journal = {"id": uuid.uuid4().hex, "phase": "installing", "installation": self.installation, "previousVersion": self.current_version, "targetVersion": self.target["version"], "targetSha256": self.target["sha256"], "agentBackupSha256": sha256(previous), "uninstallBackupSha256": sha256(uninstall), "startedAt": time.time()}
+            self.save_journal(journal)
+            self.status("updating")
+            self.check_uninstall()
+            self.load_installation()
+            systemctl("stop", AGENT_SERVICE)
+            atomic_bytes(AGENT_EXECUTABLE, source, 0o755)
+            atomic_bytes(UNINSTALL_EXECUTABLE, source, 0o755)
+            self.save_journal(journal, phase="verifying", switchedAt=time.time())
+        # The candidate takes the same lock during startup credential recovery.
+        # Never wait for its start or authenticated heartbeat while holding it.
+        systemctl("start", AGENT_SERVICE)
+        if not self.wait_healthy(journal):
+            raise UpdateError("The new agent did not report a healthy authenticated heartbeat; the previous release will be restored")
+        self.check_uninstall()
+        self.load_installation()
+        if any(sha256(read_bytes(path, MAX_SOURCE)) != journal["targetSha256"] for path in (AGENT_EXECUTABLE, UNINSTALL_EXECUTABLE)):
+            raise UpdateError("Installed agent code changed before update verification finished")
+        self.status("updated", current=journal["targetVersion"])
+        self.current_version = journal["targetVersion"]
+        self.finish_journal()
+        return "updated"
+
+    def wait_healthy(self, journal):
+        deadline = time.monotonic() + HEALTH_TIMEOUT
+        while time.monotonic() < deadline:
+            self.check_uninstall()
+            try:
+                active, pid = service_state()
+                health = read_json(AGENT_HEALTH, 4096)
+                checked = health.get("checkedAt")
+                if active and pid > 0 and type(health.get("pid")) is int and health["pid"] == pid and health.get("version") == journal["targetVersion"] and health.get("updateId") == journal["id"] and type(checked) in (float, int) and math.isfinite(checked) and journal["switchedAt"] <= checked <= time.time() + 5:
+                    return True
+            except (UpdateError, OSError, ValueError):
+                pass
+            time.sleep(2)
+        return False
+
+    def rollback(self, journal, error):
+        self.target = {"version": journal["targetVersion"], "sha256": journal["targetSha256"]}
+        self.check_uninstall()
+        self.load_installation()
+        if journal["installation"] != self.installation:
+            raise UpdateError("Agent enrollment changed; rollback needs operator review")
+        backups = [(AGENT_EXECUTABLE, "agent.backup.py", "agentBackupSha256"), (UNINSTALL_EXECUTABLE, "uninstall.backup.py", "uninstallBackupSha256")]
+        sources = []
+        for destination, name, checksum in backups:
+            source = read_bytes(UPDATE_DIR / name, MAX_SOURCE)
+            if sha256(source) != journal[checksum]:
+                raise UpdateError("Agent rollback backup failed integrity verification; repair the installation")
+            source_version(source)
+            sources.append((destination, source))
+        if source_version(sources[0][1]) != journal["previousVersion"]:
+            raise UpdateError("Agent rollback version does not match its recovery journal")
+        self.save_journal(journal, phase="rollback")
+        with self.activity_lock() as activity:
+            if activity is None:
+                raise Deferred("Agent rollback is waiting for the active operation to finish")
+            self.check_uninstall()
+            self.load_installation()
+            systemctl("stop", AGENT_SERVICE)
+            for destination, source in sources:
+                atomic_bytes(destination, source, 0o755)
+            self.save_journal(journal, rollbackSwitchedAt=time.time())
+        systemctl("start", AGENT_SERVICE)
+        rollback_health = {**journal, "targetVersion": journal["previousVersion"], "switchedAt": journal["rollbackSwitchedAt"]}
+        if not self.wait_healthy(rollback_health):
+            raise UpdateError("The previous agent release was restored but its authenticated health is unconfirmed; recovery will retry")
+        self.hold(self.target, error)
+        self.status("failed", error, current=journal["previousVersion"])
+        self.current_version = journal["previousVersion"]
+        self.finish_journal()
+        return "failed"
+
+    def run(self):
+        with exclusive_lock(UPDATE_LOCK) as updater:
+            if updater is None:
+                return "deferred"  # Preserve the active worker's status.
+            recovering = False
+            try:
+                self.check_uninstall()
+                self.load_installation()
+                if UPDATE_JOURNAL.exists() or UPDATE_JOURNAL.is_symlink():
+                    journal = self.read_journal()
+                    self.current_version = journal["previousVersion"]
+                    recovering = True
+                    return self.rollback(journal, "An interrupted agent update was rolled back to the previous release")
+                self.current_version = source_version(read_bytes(AGENT_EXECUTABLE, MAX_SOURCE))
+                self.status("checking")
+                self.target = self.manifest()
+                if version(self.target["version"]) <= version(self.current_version):
+                    return self.status("current")
+                if self.release_held():
+                    return self.status("failed", "This release previously failed verification and is held until the controller publishes a different release")
+                source = self.fetch("/api/agent/release/ec-agent.py", MAX_SOURCE)
+                if sha256(source) != self.target["sha256"]:
+                    raise IntegrityError("The agent release checksum does not match its manifest")
+                if source_version(source) != self.target["version"]:
+                    raise IntegrityError("The agent source VERSION does not match its release manifest")
+                return self.install(source)
+            except Deferred as error:
+                return self.status("deferred", error)
+            except ContactError as error:
+                return self.status("deferred", error)
+            except (UpdateError, OSError, ValueError, KeyError) as error:
+                message = str(error) if isinstance(error, UpdateError) else "Agent update could not finish; inspect the updater service"
+                if UPDATE_JOURNAL.exists():
+                    if recovering:
+                        return self.status("failed", "Agent update recovery is pending; repair the updater service before accepting new operations")
+                    try:
+                        return self.rollback(self.read_journal(), message)
+                    except Deferred as deferred:
+                        return self.status("deferred", deferred)
+                    except (UpdateError, OSError, ValueError, KeyError):
+                        return self.status("failed", "Agent update recovery is pending; repair the updater service before accepting new operations")
+                if isinstance(error, IntegrityError) and self.target:
+                    self.hold(self.target, message)
+                return self.status("failed", message)
+
+
+def main():
+    if os.geteuid() != 0:
+        raise UpdateError("Run the agent updater through its installed root service")
+    Updater().run()
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (UpdateError, OSError, ValueError, KeyError):
+        print("Electric Cherry agent updater failed; inspect its installation and update status", file=sys.stderr)
+        sys.exit(1)

static/agent/ec-agent.py

@@ -29,7 +29,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.1.0"
+VERSION = "1.2.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
@@ -39,6 +39,13 @@ AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service")
 UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
 UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service")
 UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
+UPDATE_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-update.py")
+UPDATE_UNIT = Path("/etc/systemd/system/electric-cherry-update.service")
+UPDATE_TIMER = Path("/etc/systemd/system/electric-cherry-update.timer")
+UPDATE_TIMER_ENABLED = Path("/etc/systemd/system/timers.target.wants/electric-cherry-update.timer")
+UPDATE_JOURNAL = Path("/etc/electric-cherry/update/journal.json")
+UPDATE_STATUS = Path("/etc/electric-cherry/update-status.json")
+AGENT_HEALTH = Path("/etc/electric-cherry/agent-health.json")
 ROOT_MARKER = ".electric-cherry.json"
 
 
@@ -105,7 +112,7 @@ def uninstall_root(value, identity=None):
     forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"}
     if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden:
         raise AgentError("Host data directory is unsafe to remove")
-    for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT):
+    for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT, UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER, AGENT_EXECUTABLE.with_suffix(".py.update"), UNINSTALL_EXECUTABLE.with_suffix(".py.update")):
         if installed == root or root in installed.parents:
             raise AgentError("Host data directory overlaps the agent installation")
     protected_path(root, missing=True)
@@ -499,6 +506,19 @@ class Uninstaller:
 
     def remove_data(self):
         protected_path(AGENT_CONFIG.parent, missing=True)
+        # Prevent a scheduled update from recreating agent files during removal.
+        # Stopping the worker is safe here: the installation and any backups are
+        # being removed together, and the durable decommission request blocks
+        # future update attempts.
+        if not self.job.get("updaterStopped"):
+            for unit in (UPDATE_TIMER, UPDATE_UNIT):
+                protected_file(unit, missing=True)
+                if unit.exists():
+                    if unit == UPDATE_TIMER:
+                        systemctl("disable", "--now", unit.name)
+                    else:
+                        systemctl("stop", unit.name)
+            self.save(updaterStopped=True)
         if not self.job.get("agentStopped"):
             systemctl("disable", "--now", AGENT_UNIT.name)
             self.save(agentStopped=True)
@@ -512,7 +532,7 @@ class Uninstaller:
                 raise AgentError("This platform cannot safely remove managed host files")
             shutil.rmtree(root)
             sync_parent(root)
-        for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT):
+        for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER, AGENT_EXECUTABLE.with_suffix(".py.update"), UNINSTALL_EXECUTABLE.with_suffix(".py.update")):
             protected_file(path, missing=True)
             path.unlink(missing_ok=True)
             sync_parent(path)
@@ -591,6 +611,74 @@ class Agent:
         except (AgentError, OSError, ValueError):
             return False
 
+    def can_update(self):
+        try:
+            if Path(self.config_path) != AGENT_CONFIG:
+                return False
+            uninstall_ownership(self.config)
+            for path in (UPDATE_EXECUTABLE, UPDATE_UNIT, UPDATE_TIMER):
+                protected_file(path)
+            protected_path(UPDATE_TIMER_ENABLED.parent)
+            return UPDATE_TIMER_ENABLED.is_symlink() and UPDATE_TIMER_ENABLED.resolve() == UPDATE_TIMER
+        except (AgentError, OSError, ValueError):
+            return False
+
+    def update_inventory(self):
+        result = {"enabled": self.can_update(), "state": "current", "currentVersion": VERSION}
+        if Path(self.config_path) != AGENT_CONFIG:
+            return result
+        try:
+            protected_file(UPDATE_STATUS, missing=True)
+            if not UPDATE_STATUS.exists():
+                return result
+            if UPDATE_STATUS.stat().st_size > 16384:
+                raise ValueError("Invalid update status")
+            saved = json.loads(UPDATE_STATUS.read_text())
+            if saved.get("state") in ("current", "checking", "deferred", "updating", "updated", "failed"):
+                result["state"] = saved["state"]
+            for key in ("targetVersion", "error"):
+                if isinstance(saved.get(key), str):
+                    result[key] = saved[key][:2000 if key == "error" else 80]
+            checked_at = saved.get("checkedAt")
+            if isinstance(checked_at, (int, float)) and not isinstance(checked_at, bool):
+                result["checkedAt"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(checked_at))
+            elif isinstance(checked_at, str):
+                result["checkedAt"] = checked_at[:80]
+        except (AgentError, OSError, ValueError, TypeError, AttributeError, OverflowError):
+            result.update(state="failed", error="Agent update status is unavailable; inspect the update service.")
+        return result
+
+    def update_pending(self):
+        # A persisted transaction survives crashes and is cleared only by the
+        # independent worker after health confirmation or a completed rollback.
+        return Path(self.config_path) == AGENT_CONFIG and (UPDATE_JOURNAL.exists() or UPDATE_JOURNAL.is_symlink())
+
+    @contextlib.contextmanager
+    def activity(self):
+        path = self.root / "agent" / "activity.lock"
+        descriptor = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+        try:
+            try:
+                fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError:
+                yield False
+                return
+            yield True
+        finally:
+            os.close(descriptor)
+
+    def record_health(self):
+        if Path(self.config_path) != AGENT_CONFIG or not self.can_update():
+            return
+        value = {"version": VERSION, "pid": os.getpid(), "checkedAt": time.time()}
+        protected_file(AGENT_HEALTH, missing=True)
+        if self.update_pending():
+            protected_file(UPDATE_JOURNAL)
+            journal = json.loads(UPDATE_JOURNAL.read_text())
+            if isinstance(journal.get("id"), str):
+                value["updateId"] = journal["id"]
+        atomic_json(AGENT_HEALTH, value)
+
     def decommission(self, operation):
         payload = operation.get("payload", {})
         if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]:
@@ -733,7 +821,9 @@ class Agent:
             for line in Path("/etc/os-release").read_text().splitlines():
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
-        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
+        update = self.update_inventory()
+        capabilities = (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
         health = self.inspect(instance_id, generation)
@@ -1462,29 +1552,21 @@ class Agent:
         host = response.get("host", {})
         if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int):
             self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]}
+        # Written only after the controller accepted this process's authenticated
+        # heartbeat. Service liveness alone cannot prove an update works.
+        self.record_health()
 
     def heartbeat_loop(self):
         while not self.stopping.wait(15):
             try:
                 self.heartbeat()
-            except AgentError:
+            except (AgentError, OSError, ValueError):
                 pass
 
-    def run(self):
-        if self.config.get("pendingCredential"):
-            self.recover_credential()
-        # Obtain the configured host port range before accepting any deployment.
-        # Enrollment's default values must never race the first heartbeat.
-        while not self.stopping.is_set():
-            try:
-                self.heartbeat()
-                break
-            except AgentError:
-                self.stopping.wait(3)
-        heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
-        heartbeats.start()
-        while not self.stopping.is_set():
-            try:
+    def poll_operations(self):
+        with self.activity() as acquired:
+            if not acquired or self.stopping.is_set() or self.update_pending():
+                return
             if self.config.get("pendingCredential"):
                 self.recover_credential()
             response = self.request("GET", "/api/agent/operations")
@@ -1492,10 +1574,33 @@ class Agent:
                 if self.stopping.is_set():
                     break
                 self.current_operation = operation["id"]
+                try:
                     accepted, _ = self.journal.accept(operation)
                     self.report(accepted)
                     self.report(self.execute(operation, report_running=True))
+                finally:
                     self.current_operation = None
+
+    def run(self):
+        # Obtain the configured host port range before accepting any deployment.
+        # Enrollment's default values must never race the first heartbeat.
+        while not self.stopping.is_set():
+            try:
+                with self.activity() as acquired:
+                    if not acquired:
+                        self.stopping.wait(3)
+                        continue
+                    if self.config.get("pendingCredential"):
+                        self.recover_credential()
+                self.heartbeat()
+                break
+            except (AgentError, OSError, ValueError):
+                self.stopping.wait(3)
+        heartbeats = threading.Thread(target=self.heartbeat_loop, daemon=True)
+        heartbeats.start()
+        while not self.stopping.is_set():
+            try:
+                self.poll_operations()
             except (AgentError, OSError, ValueError, KeyError):
                 # Logs deliberately omit URLs, tokens, payloads and raw errors.
                 print("Host controller contact unavailable; existing containers continue running", file=sys.stderr)

static/agent/install.sh

@@ -8,6 +8,8 @@ controller=''
 enrollment_token=''
 data_dir='/var/lib/electric-cherry'
 reenroll=''
+existing_enrollment=false
+existing_data_root=false
 while [ "$#" -gt 0 ]; do
     case "$1" in
         --controller) controller="${2:?Missing controller URL}"; shift 2 ;;
@@ -31,7 +33,16 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; }
 
 export DEBIAN_FRONTEND=noninteractive
 apt-get update -qq
-apt-get install -y --no-install-recommends ca-certificates curl python3
+apt-get install -y --no-install-recommends ca-certificates curl python3 util-linux
+installation_preflight() {
+    if [ "$existing_enrollment" = true ] && [ ! -f /etc/electric-cherry/agent.json ]; then
+        echo 'The existing enrollment disappeared during installation. Stop and inspect host deletion before trying again.' >&2
+        return 1
+    fi
+    if [ "$existing_data_root" = true ] && [ ! -d "$data_dir" ]; then
+        echo 'The existing data directory disappeared during installation. Stop and inspect host deletion before trying again.' >&2
+        return 1
+    fi
 python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY'
 # INSTALLATION_PREFLIGHT: this runs before touching managed directories or services.
 import json, pathlib, re, stat, sys
@@ -51,9 +62,14 @@ if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root)
     raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.')
 installed_paths = [config,
     pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-agent.py.update'),
     pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py.update'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-update.py'),
     pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'),
-    pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service')]
+    pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-update.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-update.timer')]
 if any(root == path or root in path.parents for path in installed_paths):
     raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.')
 
@@ -66,10 +82,13 @@ def trusted_directories(path):
         if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
             raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.')
 
-def trusted_json(path):
+def trusted_file(path):
     info = path.lstat()
     if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
         raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.')
+
+def trusted_json(path):
+    trusted_file(path)
     value = json.loads(path.read_text())
     if not isinstance(value, dict):
         raise SystemExit('Existing installation metadata is invalid.')
@@ -89,6 +108,13 @@ if mountinfo.exists():
 request = config.with_name('decommission.json')
 if request.exists() or request.is_symlink():
     raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.')
+trusted_directories(config.parent / 'update')
+journal = config.parent / 'update' / 'journal.json'
+if journal.exists() or journal.is_symlink():
+    raise SystemExit('An automatic update needs recovery. Let electric-cherry-update.service finish or repair that update before rerunning the installer.')
+update_lock = config.with_name('update.lock')
+if update_lock.exists() or update_lock.is_symlink():
+    trusted_file(update_lock)
 saved = None
 if config.exists() or config.is_symlink():
     saved = trusted_json(config)
@@ -108,6 +134,62 @@ elif root.exists() and any(root.iterdir()):
         raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.')
 for directory in ('agent', 'instances', 'libraries'):
     trusted_directories(root / directory)
+activity_lock = root / 'agent' / 'activity.lock'
+if activity_lock.exists() or activity_lock.is_symlink():
+    trusted_file(activity_lock)
+PY
+}
+if [ -f /etc/electric-cherry/agent.json ]; then existing_enrollment=true; fi
+if [ -d "$data_dir" ]; then existing_data_root=true; fi
+installation_preflight
+temporary_dir=$(mktemp -d)
+updater_timer_was_active=false
+update_lock_held=false
+activity_lock_held=false
+installer_finished=false
+installation_cleanup() {
+    local ec_install_status=$?
+    trap - EXIT
+    if [ "$activity_lock_held" = true ]; then
+        flock --unlock 8 || true
+        exec 8>&-
+    fi
+    if [ "$update_lock_held" = true ]; then
+        flock --unlock 9 || true
+        exec 9>&-
+    fi
+    # Recovery must remain scheduled when an upgrade or its second preflight
+    # fails. Release the worker lock before restarting the prior active timer.
+    if [ "$installer_finished" != true ] && [ "$updater_timer_was_active" = true ]; then
+        systemctl start electric-cherry-update.timer >/dev/null 2>&1 || true
+    fi
+    rm -rf "$temporary_dir" || true
+    exit "$ec_install_status"
+}
+trap installation_cleanup EXIT
+controller="${controller%/}"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py"
+curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent-update.py" -o "$temporary_dir/ec-agent-update.py"
+python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" "$temporary_dir/ec-agent-update.py" <<'PY'
+# INSTALLATION_RELEASE: check both downloads before installing or executing either.
+import ast, hashlib, json, pathlib, re, sys
+manifest = json.loads(pathlib.Path(sys.argv[1]).read_text())
+agent = pathlib.Path(sys.argv[2]).read_bytes()
+updater = pathlib.Path(sys.argv[3]).read_bytes()
+if hashlib.sha256(agent).hexdigest() != manifest.get('sha256'):
+    raise SystemExit('Agent release checksum verification failed.')
+if hashlib.sha256(updater).hexdigest() != manifest.get('updaterSha256'):
+    raise SystemExit('Updater release checksum verification failed.')
+if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1 or manifest.get('updaterProtocolVersion') != 1:
+    raise SystemExit('Unsupported agent or updater release manifest.')
+versions = [node.value.value for node in ast.parse(agent).body
+            if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == 'VERSION' for target in node.targets)
+            and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str)]
+if len(versions) != 1 or versions[0] != manifest.get('version') or not re.fullmatch(r'(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)', versions[0]):
+    raise SystemExit('Agent release version does not match its manifest.')
+compile(agent, 'ec-agent.py', 'exec')
+compile(updater, 'ec-agent-update.py', 'exec')
 PY
 if ! command -v docker >/dev/null; then
     if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then
@@ -118,29 +200,37 @@ if ! command -v docker >/dev/null; then
     systemctl enable --now docker
 fi
 docker info >/dev/null 2>&1 || { echo 'Docker is installed but unavailable. Start or repair it, then rerun.' >&2; exit 1; }
-install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances"
-install -d -m 0700 "$data_dir/agent" /etc/electric-cherry
-available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}')
-[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; }
-temporary_dir=$(mktemp -d)
-trap 'rm -rf "$temporary_dir"' EXIT
-controller="${controller%/}"
-curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/manifest.json" -o "$temporary_dir/manifest.json"
-curl --proto '=https' --tlsv1.2 --fail --silent --show-error "$controller/api/agent/release/ec-agent.py" -o "$temporary_dir/ec-agent.py"
-python3 - "$temporary_dir/manifest.json" "$temporary_dir/ec-agent.py" <<'PY'
-import hashlib, json, pathlib, sys
-manifest = json.loads(pathlib.Path(sys.argv[1]).read_text())
-actual = hashlib.sha256(pathlib.Path(sys.argv[2]).read_bytes()).hexdigest()
-if actual != manifest.get('sha256'):
-    raise SystemExit('Agent release checksum verification failed.')
-if manifest.get('platform') != 'debian-amd64' or manifest.get('protocolVersion') != 1:
-    raise SystemExit('Unsupported agent release manifest.')
-PY
+install -d -m 0700 /etc/electric-cherry
+# Wait for an already-running updater to finish; never interrupt its rollback.
+# The same lock also prevents a timer firing while files are being installed.
+if systemctl cat electric-cherry-update.timer >/dev/null 2>&1; then
+    if systemctl is-active --quiet electric-cherry-update.timer; then
+        updater_timer_was_active=true
+    fi
+    systemctl stop electric-cherry-update.timer
+fi
+exec 9>/etc/electric-cherry/update.lock
+flock --exclusive 9
+update_lock_held=true
+installation_preflight
+# Serialize with running operations, especially a deletion request that could
+# otherwise start between preflight and stopping the main agent.
+if [ -d "$data_dir/agent" ]; then
+    exec 8>"$data_dir/agent/activity.lock"
+    flock --exclusive 8
+    activity_lock_held=true
+    installation_preflight
+fi
 # Stop only the agent while replacing its own executable/journal owner. Existing
 # game containers keep running under Docker throughout installation and updates.
 if systemctl cat electric-cherry-agent.service >/dev/null 2>&1; then
     systemctl stop electric-cherry-agent.service
 fi
+installation_preflight
+install -d -m 0755 "$data_dir" "$data_dir/libraries" "$data_dir/instances"
+install -d -m 0700 "$data_dir/agent"
+available_kb=$(df -Pk "$data_dir" | awk 'NR==2 {print $4}')
+[ "$available_kb" -ge 1048576 ] || { echo 'At least 1 GiB of available game storage is required.' >&2; exit 1; }
 install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py 2>/dev/null || {
     install -d -m 0755 /usr/local/libexec
     install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py
@@ -148,6 +238,7 @@ install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-
 # A separate worker survives stopping/removing the agent service. Its only
 # entry point is a fixed, authenticated deletion request prepared by the agent.
 install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py
+install -m 0755 "$temporary_dir/ec-agent-update.py" /usr/local/libexec/electric-cherry-update.py
 printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token"
 unset enrollment_token
 python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"}
@@ -215,8 +306,41 @@ UMask=0077
 [Install]
 WantedBy=multi-user.target
 EOF
+cat > /etc/systemd/system/electric-cherry-update.service <<'EOF'
+[Unit]
+Description=Update the Electric Cherry Host Agent
+After=network-online.target docker.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-update.py
+TimeoutStartSec=0
+UMask=0077
+EOF
+cat > /etc/systemd/system/electric-cherry-update.timer <<'EOF'
+[Unit]
+Description=Check for Electric Cherry Host Agent Updates
+
+[Timer]
+OnBootSec=2min
+OnUnitInactiveSec=5min
+RandomizedDelaySec=30s
+AccuracySec=1s
+Unit=electric-cherry-update.service
+
+[Install]
+WantedBy=timers.target
+EOF
 systemctl daemon-reload
 systemctl enable electric-cherry-agent
+if [ "$activity_lock_held" = true ]; then
+    flock --unlock 8
+    exec 8>&-
+    activity_lock_held=false
+fi
 systemctl restart electric-cherry-agent
 systemctl is-active --quiet electric-cherry-agent || { echo 'Agent service did not start. Inspect journalctl -u electric-cherry-agent.' >&2; exit 1; }
-echo 'Electric Cherry host agent installed and enrolled. Host readiness is shown in Hosts.'
+systemctl enable --now electric-cherry-update.timer
+installer_finished=true
+echo 'Electric Cherry host agent installed and enrolled. Automatic agent updates are enabled; host readiness is shown in Hosts.'

static/agent/manifest.json

@@ -1,6 +1,8 @@
 {
-  "version": "1.1.0",
+  "version": "1.2.0",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "67046a3d81f251e148f2323cade255ff584a622750a8c05ee27f96c70e09fcc7"
+  "sha256": "c0c08bc50daf09f4e6e141d4119cee68973b47c29845b2f3620e4579d9cd7daa",
+  "updaterProtocolVersion": 1,
+  "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }

tests/unit/agent-update-inventory.test.ts

@@ -0,0 +1,15 @@
+import { describe, expect, it } from 'vitest';
+import { validateInventory } from '../../src/lib/server/hosts/index.js';
+
+const update = { enabled: true, state: 'updated', currentVersion: '1.2.0', targetVersion: '1.2.0', checkedAt: '2026-10-04T12:00:00Z' };
+describe('agent update inventory', () => {
+    it('accepts legacy inventory and only publishes recognized update display fields', () => {
+        expect(validateInventory({ protocolVersion: 1 })).toEqual({ protocolVersion: 1 });
+        expect(validateInventory({ agentUpdate: { ...update, credential: 'private', journal: { backups: 'local-path' } } }).agentUpdate).toEqual(update);
+    });
+    it.each([null, [], 'updated', { ...update, enabled: 'yes' }, { ...update, state: '<script>' },
+        { ...update, currentVersion: {} }, { ...update, targetVersion: 'invalid' }, { ...update, checkedAt: 'invalid' },
+        { ...update, error: {} }, { ...update, error: 'x'.repeat(2001) }])('rejects invalid update metadata %j', agentUpdate => {
+        expect(() => validateInventory({ agentUpdate })).toThrow('Invalid agent update inventory');
+    });
+});

tests/unit/auth-guard.test.ts

@@ -185,10 +185,10 @@ describe('machine and route identity boundaries', () => {
         expect((await rawConfig.invoke()).status).toBe(403);
     });
     it('exempts only static assets on unresolved routes and does not use arbitrary file extensions as public access', async () => {
-        for (const path of ['/_app/immutable/start.js', '/favicon.svg', '/agent/ec-agent.py', '/agent/install.sh', '/agent/manifest.json']) {
+        for (const path of ['/_app/immutable/start.js', '/favicon.svg', '/agent/ec-agent.py', '/agent/ec-agent-update.py', '/agent/install.sh', '/agent/manifest.json']) {
             const call = requestFor(null, { path }); expect((await call.invoke()).status).toBe(200); expect(call.resolve).toHaveBeenCalledOnce();
         }
-        for (const path of ['/private.json', '/secrets.css', '/settings/users/__data.json']) {
+        for (const path of ['/private.json', '/secrets.css', '/settings/users/__data.json', '/agent/ec-agent-update.py.bak', '/agent/update-status.json']) {
             const call = requestFor(null, { path }); await expect(call.invoke()).rejects.toMatchObject({ status: 303 }); expect(call.resolve).not.toHaveBeenCalled();
         }
         const write = requestFor(null, { path: '/_app/anything.js', method: 'POST', body: {} });

tests/unit/host-agent-release.test.ts

@@ -0,0 +1,75 @@
+import { resolve } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const files = vi.hoisted(() => ({ readFile: vi.fn() }));
+vi.mock('node:fs/promises', () => ({ readFile: files.readFile }));
+import { agentReleaseAsset } from '../../src/lib/server/hosts/releases.js';
+import { GET } from '../../src/routes/api/agent/release/[file]/+server.js';
+
+beforeEach(() => { files.readFile.mockReset(); vi.stubEnv('NODE_ENV', 'production'); });
+afterEach(() => { vi.unstubAllEnvs(); });
+
+describe('public agent release artifacts', () => {
+    it('keeps every development artifact on the fresh static release when an older build also exists', async () => {
+        vi.stubEnv('NODE_ENV', 'development');
+        files.readFile.mockImplementation(async (path: string) => path.startsWith(resolve('static/agent')) ? 'fresh-release' : 'old-build');
+        for (const file of ['install.sh', 'ec-agent.py', 'ec-agent-update.py', 'manifest.json'] as const) {
+            expect(await agentReleaseAsset(file)).toBe('fresh-release');
+        }
+        expect(files.readFile.mock.calls.map(call => call[0])).toEqual([
+            resolve('static/agent/install.sh'), resolve('static/agent/ec-agent.py'),
+            resolve('static/agent/ec-agent-update.py'), resolve('static/agent/manifest.json')
+        ]);
+    });
+
+    it('does not fill a missing development artifact from an unrelated older build', async () => {
+        vi.stubEnv('NODE_ENV', 'development');
+        const missing = Object.assign(new Error('Not packaged yet'), { code: 'ENOENT' });
+        files.readFile.mockRejectedValueOnce(missing).mockResolvedValue('old-build');
+        await expect(agentReleaseAsset('ec-agent-update.py')).rejects.toBe(missing);
+        expect(files.readFile.mock.calls).toEqual([[resolve('static/agent/ec-agent-update.py'), 'utf8']]);
+    });
+
+    it('uses the packaged production directory for every artifact even when source assets exist', async () => {
+        files.readFile.mockImplementation(async (path: string) => path.startsWith(resolve('build/client/agent')) ? 'packaged-release' : 'source-release');
+        for (const file of ['install.sh', 'ec-agent.py', 'ec-agent-update.py', 'manifest.json'] as const) {
+            expect(await agentReleaseAsset(file)).toBe('packaged-release');
+        }
+        expect(files.readFile.mock.calls.map(call => call[0])).toEqual([
+            resolve('build/client/agent/install.sh'), resolve('build/client/agent/ec-agent.py'),
+            resolve('build/client/agent/ec-agent-update.py'), resolve('build/client/agent/manifest.json')
+        ]);
+    });
+
+    it.each(['ec-agent.py', 'ec-agent-update.py', 'manifest.json'])('serves the packaged %s without a browser session or host credential', async file => {
+        const content = file.endsWith('.json') ? '{"version":"1.2.0","updaterProtocolVersion":1}' : '# verified release source\n';
+        files.readFile.mockResolvedValue(content);
+        const response = await GET({ params: { file } });
+        expect(response.status).toBe(200);
+        expect(await response.text()).toBe(content);
+        expect(response.headers.get('cache-control')).toBe('no-cache');
+        expect(response.headers.get('content-type')).toContain(file.endsWith('.json') ? 'application/json' : 'text/plain');
+        expect(files.readFile.mock.calls).toEqual([[resolve('build/client/agent', file), 'utf8']]);
+    });
+
+    it('serves the updater from development assets when no packaged asset exists', async () => {
+        files.readFile.mockRejectedValueOnce(Object.assign(new Error('missing'), { code: 'ENOENT' })).mockResolvedValueOnce('# updater\n');
+        expect(await agentReleaseAsset('ec-agent-update.py')).toBe('# updater\n');
+        expect(files.readFile.mock.calls).toEqual([
+            [resolve('build/client/agent/ec-agent-update.py'), 'utf8'],
+            [resolve('static/agent/ec-agent-update.py'), 'utf8']
+        ]);
+    });
+
+    it('does not conceal a damaged packaged release by falling back to a different artifact', async () => {
+        const failure = Object.assign(new Error('permission denied'), { code: 'EACCES' });
+        files.readFile.mockRejectedValue(failure);
+        await expect(agentReleaseAsset('ec-agent-update.py')).rejects.toBe(failure);
+        expect(files.readFile).toHaveBeenCalledTimes(1);
+    });
+
+    it.each(['../ec-agent.py', '%2e%2e%2fmanifest.json', 'agent-health.json', 'update-status.json', 'config.json', 'install.sh', 'ec-agent-update.py/anything'])('rejects non-release file %s before reading disk', async file => {
+        await expect(GET({ params: { file } })).rejects.toMatchObject({ status: 404 });
+        expect(files.readFile).not.toHaveBeenCalled();
+    });
+});

tests/unit/host-agent-update-status.test.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it } from 'vitest';
+import { hostAgentUpdateLabel, hostAgentUpdateMessage, hostAgentUpdatesEnabled, type HostInventory } from '../../src/lib/hosts.js';
+
+const installed: HostInventory = {
+    agentVersion: '1.2.0', capabilities: ['host.decommission', 'agent.update'],
+    agentUpdate: { enabled: true, state: 'current', currentVersion: '1.2.0', checkedAt: '2026-10-04T12:00:00Z' }
+};
+
+describe('host automatic update status', () => {
+    it('requires the installed updater capability instead of inferring setup from agent version', () => {
+        for (const inventory of [
+            {}, { agentVersion: '1.2.0' },
+            { ...installed, capabilities: ['host.decommission'] },
+            { ...installed, agentUpdate: { ...installed.agentUpdate!, enabled: false } }
+        ]) {
+            expect(hostAgentUpdatesEnabled(inventory)).toBe(false);
+            expect(hostAgentUpdateLabel(inventory)).toBe('Setup required');
+            expect(hostAgentUpdateMessage(inventory)).toMatch(/installer once/);
+        }
+        expect(hostAgentUpdatesEnabled(installed)).toBe(true);
+        expect(hostAgentUpdateLabel(installed)).toBe('Current');
+    });
+
+    it('does not claim an unchecked fresh installation has verified the latest release', () => {
+        const fresh = { ...installed, agentUpdate: { ...installed.agentUpdate!, checkedAt: undefined } };
+        expect(hostAgentUpdateLabel(fresh)).toBe('Awaiting first check');
+        expect(hostAgentUpdateMessage(fresh)).toContain('will check');
+        expect(hostAgentUpdateMessage(installed)).toContain('last check');
+    });
+
+    it('distinguishes a deferred check from a failed update without assuming every hold is an active operation', () => {
+        const deferred = { ...installed, agentUpdate: { ...installed.agentUpdate!, state: 'deferred' as const, targetVersion: '1.3.0' } };
+        expect(hostAgentUpdateLabel(deferred)).toBe('Deferred');
+        expect(hostAgentUpdateMessage(deferred)).toContain('reported condition to clear');
+        expect(hostAgentUpdateMessage(deferred)).toContain('checks again automatically');
+        const failed = { ...installed, agentUpdate: { ...installed.agentUpdate!, state: 'failed' as const, error: 'Checksum mismatch' } };
+        expect(hostAgentUpdateLabel(failed)).toBe('Needs attention');
+        expect(hostAgentUpdateMessage(failed)).toContain('did not complete');
+    });
+
+    it('describes connection verification until the updater confirms success', () => {
+        const updating = { ...installed, agentUpdate: { ...installed.agentUpdate!, state: 'updating' as const, targetVersion: '1.3.0' } };
+        expect(hostAgentUpdateLabel(updating)).toBe('Updating');
+        expect(hostAgentUpdateMessage(updating)).toContain('checking that the new process connects');
+        const updated = { ...installed, agentUpdate: { ...installed.agentUpdate!, state: 'updated' as const, currentVersion: '1.3.0' } };
+        expect(hostAgentUpdateLabel(updated)).toBe('Updated');
+        expect(hostAgentUpdateMessage(updated)).toContain('connected successfully');
+    });
+});