Remove host files and uninstall agent when deleting hosts

agent/ec_agent.py

@@ -29,10 +29,17 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.0.0"
+VERSION = "1.1.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
+AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
+AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
+AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service")
+UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
+UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service")
+UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
+ROOT_MARKER = ".electric-cherry.json"
 
 
 class AgentError(Exception):
@@ -47,6 +54,97 @@ class Rejected(AgentError):
     """A definitive precondition failure, before any destructive action."""
 
 
+def systemctl(*args):
+    try:
+        result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
+    except (OSError, subprocess.TimeoutExpired):
+        raise AgentError("Host service management is unavailable") from None
+    if result.returncode:
+        raise AgentError("Host service management did not complete")
+
+
+def protected_path(path, missing=False):
+    """Deletion authority must never come from a runtime-writable path."""
+    path = Path(path)
+    for current in (*reversed(path.parents), path):
+        try:
+            info = current.lstat()
+        except FileNotFoundError:
+            if missing:
+                continue
+            raise AgentError("Host uninstall files are unavailable; reinstall the agent") from None
+        if stat_module.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+            raise AgentError("Host uninstall paths must be owned and writable only by root")
+    return path
+
+
+def protected_file(path, missing=False):
+    path = protected_path(path, missing=missing)
+    try:
+        if not stat_module.S_ISREG(path.lstat().st_mode):
+            raise AgentError("Host uninstall metadata must be a regular file")
+    except FileNotFoundError:
+        if not missing:
+            raise
+    return path
+
+
+def sync_parent(path):
+    descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def uninstall_root(value, identity=None):
+    """Accept only a canonical, dedicated directory with no nested mounts."""
+    if not isinstance(value, str) or not re.fullmatch(r"/[A-Za-z0-9/_.-]+", value):
+        raise AgentError("Host data directory is unsafe to remove")
+    root = Path(value)
+    forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"}
+    if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden:
+        raise AgentError("Host data directory is unsafe to remove")
+    for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT):
+        if installed == root or root in installed.parents:
+            raise AgentError("Host data directory overlaps the agent installation")
+    protected_path(root, missing=True)
+    if root.exists():
+        info = root.stat()
+        if not stat_module.S_ISDIR(info.st_mode):
+            raise AgentError("Host data directory is not a directory")
+        if identity and [info.st_dev, info.st_ino] != identity:
+            raise AgentError("Host data directory changed since deletion was authorized")
+    try:
+        # ProtectSystem/ReadWritePaths give the running agent a private bind
+        # mount at its data directory. Inspect the host systemd namespace,
+        # otherwise every legitimate installation appears to be a mount root.
+        mount_lines = Path("/proc/1/mountinfo").read_text().splitlines()
+    except FileNotFoundError:
+        mount_lines = []  # Non-Linux tests; the installed service requires Linux.
+    for line in mount_lines:
+        fields = line.split()
+        if len(fields) < 5:
+            continue
+        mount = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), fields[4])
+        if mount == value or mount.startswith(value + "/"):
+            raise AgentError("Unmount filesystems inside the host data directory before deletion")
+    return root
+
+
+def uninstall_ownership(config):
+    root = uninstall_root(config.get("root"))
+    marker = protected_file(root / ROOT_MARKER)
+    try:
+        saved = json.loads(marker.read_text())
+    except (OSError, ValueError):
+        raise AgentError("Host data ownership is unavailable; reinstall the agent") from None
+    expected = {"version": 1, "root": str(root), "hostId": config.get("hostId"), "controller": config.get("controller", "").rstrip("/")}
+    if saved != expected or not expected["hostId"]:
+        raise AgentError("Host data ownership does not match this enrollment")
+    return root
+
+
 def digest(data):
     return hashlib.sha256(data).hexdigest()
 
@@ -275,7 +373,7 @@ class Docker:
             raise AgentError("Docker command unavailable or timed out") from None
         if result.returncode:
             # Never include arbitrary Docker stderr: create errors may echo secrets.
-            if tolerate_missing and (b"No such object" in (result.stderr or b"") or b"No such container" in (result.stderr or b"")):
+            if tolerate_missing and any(message in (result.stderr or b"") for message in (b"No such object", b"No such container", b"No such image")):
                 return None
             raise AgentError("Docker rejected the requested operation")
         return result.stdout.decode("utf-8", "replace")
@@ -285,6 +383,188 @@ class Docker:
         return json.loads(raw)[0] if raw is not None else None
 
 
+class Uninstaller:
+    """A separate systemd service owns cleanup after stopping the host agent.
+
+    Its root-only request and executable stay on disk until the controller
+    acknowledges data removal. Completion is reported only after removing those
+    final files. A lost final response can be retried from memory; a power loss
+    in that last window deliberately leaves controller status unconfirmed.
+    """
+
+    def __init__(self, request_path, docker=None):
+        if Path(request_path) != UNINSTALL_REQUEST:
+            raise AgentError("Invalid host uninstall request location")
+        protected_file(UNINSTALL_REQUEST)
+        self.job = json.loads(UNINSTALL_REQUEST.read_text())
+        self.docker = docker or Docker()
+        self.root = uninstall_root(self.job.get("root"), self.job.get("rootIdentity"))
+        host_id = identifier(self.job.get("hostId"))
+        operation_id = identifier(self.job.get("operationId"))
+        parsed = urllib.parse.urlsplit(self.job.get("controller", ""))
+        if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise AgentError("Host uninstall controller must use HTTPS")
+        if self.job.get("receiptUrl") != "/api/agent/decommission/" + operation_id or not isinstance(self.job.get("receiptToken"), str) or not self.job["receiptToken"]:
+            raise AgentError("Invalid host uninstall receipt")
+        if not isinstance(self.job.get("rootIdentity"), list) or len(self.job["rootIdentity"]) != 2 or not all(isinstance(value, int) and not isinstance(value, bool) for value in self.job["rootIdentity"]):
+            raise AgentError("Invalid host data ownership proof")
+        if self.root.exists():
+            # The authorized device/inode is persisted before deletion so a
+            # crash halfway through rmtree does not require a deleted marker.
+            if not self.job.get("removalStarted"):
+                uninstall_ownership({"root": str(self.root), "controller": self.job["controller"], "hostId": host_id})
+
+    def save(self, **changes):
+        self.job.update(changes)
+        atomic_json(UNINSTALL_REQUEST, self.job)
+
+    def receipt(self, stage, error=None):
+        payload = {"stage": stage}
+        if error:
+            payload["error"] = error
+        request = urllib.request.Request(self.job["controller"].rstrip("/") + self.job["receiptUrl"], data=json.dumps(payload).encode(), headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.job["receiptToken"], "User-Agent": "electric-cherry-agent/" + VERSION}, method="POST")
+        class NoRedirect(urllib.request.HTTPRedirectHandler):
+            def redirect_request(self, req, fp, code, msg, hdrs, newurl):
+                raise AgentError("Controller redirects are not allowed")
+        try:
+            with urllib.request.build_opener(NoRedirect()).open(request, timeout=35) as response:
+                if json.load(response).get("acknowledged") is not True:
+                    raise AgentError("Controller has not acknowledged host removal")
+        except (OSError, ValueError):
+            raise AgentError("Host removal receipt is waiting for controller contact") from None
+
+    def containers(self):
+        ids = self.docker.call("container", "ls", "--all", "--quiet", "--no-trunc").splitlines()
+        values = []
+        for container_id in ids:
+            info = self.docker.inspect(container_id)
+            if info:
+                values.append(info)
+        return values
+
+    def owned_container(self, info):
+        labels = info.get("Config", {}).get("Labels", {}) or {}
+        if labels.get(LABEL + ".managed") != "true":
+            return False
+        if labels.get(LABEL + ".host") not in (None, self.job["hostId"]):
+            return False
+        try:
+            instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", "")))
+        except (AgentError, ValueError, TypeError):
+            return False
+        expected = str(self.root / "instances" / instance_id / str(generation) / "app")
+        # Legacy agents did not label the host; its exact managed bind mount
+        # proves ownership without touching another installation's containers.
+        return any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", []))
+
+    def uses_root(self, info):
+        for mount in info.get("Mounts", []):
+            if mount.get("Type") != "bind":
+                continue
+            source = Path(mount.get("Source", "/")).resolve()
+            if source == self.root or self.root in source.parents:
+                return True
+        return False
+
+    def remove_docker_resources(self):
+        containers = self.containers()
+        if any(self.uses_root(info) and not self.owned_container(info) for info in containers):
+            raise AgentError("An unrelated container uses the host data directory; detach it before retrying deletion")
+        owned = [info for info in containers if self.owned_container(info)]
+        if any(info.get("State", {}).get("Running") is not False or info.get("State", {}).get("Status") not in ("created", "exited", "dead") for info in owned):
+            raise AgentError("A managed game container is still running or its state is unknown; retire it before deleting this host")
+        for info in owned:
+            # Do not force removal: Docker must also reject a container that
+            # starts again after inspection rather than interrupting players.
+            self.docker.call("container", "rm", "--volumes", info["Id"], timeout=120, tolerate_missing=True)
+        remaining = self.containers()
+        if any(self.uses_root(info) for info in remaining):
+            raise AgentError("Containers still use the host data directory")
+        # Never prune Docker globally. Only remove this agent's recorded image
+        # references, and preserve images used by any unrelated container.
+        used_images = {info.get("Image") for info in remaining}
+        images = self.job.get("runtimeImages", {})
+        if not isinstance(images, dict):
+            raise AgentError("Managed runtime image ownership is unavailable")
+        for reference, expected_id in images.items():
+            if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,511}", reference) or not isinstance(expected_id, str):
+                raise AgentError("Managed runtime image ownership is invalid")
+            raw = self.docker.call("image", "inspect", reference, tolerate_missing=True)
+            if raw is None:
+                continue
+            found = json.loads(raw)
+            if not found or found[0].get("Id") != expected_id or expected_id in used_images:
+                continue
+            self.docker.call("image", "rm", reference, tolerate_missing=True)
+
+    def remove_data(self):
+        protected_path(AGENT_CONFIG.parent, missing=True)
+        if not self.job.get("agentStopped"):
+            systemctl("disable", "--now", AGENT_UNIT.name)
+            self.save(agentStopped=True)
+        self.remove_docker_resources()
+        root = uninstall_root(str(self.root), self.job["rootIdentity"])
+        # This checkpoint also authorizes safe continuation if a power loss
+        # removes the ownership marker before the rest of the tree.
+        self.save(removalStarted=True)
+        if root.exists():
+            if not shutil.rmtree.avoids_symlink_attacks:
+                raise AgentError("This platform cannot safely remove managed host files")
+            shutil.rmtree(root)
+            sync_parent(root)
+        for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT):
+            protected_file(path, missing=True)
+            path.unlink(missing_ok=True)
+            sync_parent(path)
+        systemctl("daemon-reload")
+        self.save(dataRemoved=True)
+
+    def remove_helper(self):
+        # Disable does not stop this process: it must outlive its own unit and
+        # executable to deliver the final receipt after the files are gone.
+        protected_file(UNINSTALL_UNIT, missing=True)
+        if UNINSTALL_UNIT.exists():
+            systemctl("disable", UNINSTALL_UNIT.name)
+            UNINSTALL_UNIT.unlink()
+        sync_parent(UNINSTALL_UNIT)
+        systemctl("daemon-reload")
+        protected_file(UNINSTALL_EXECUTABLE, missing=True)
+        UNINSTALL_EXECUTABLE.unlink(missing_ok=True)
+        sync_parent(UNINSTALL_EXECUTABLE)
+        protected_path(UNINSTALL_REQUEST.parent, missing=True)
+        if UNINSTALL_REQUEST.parent.exists():
+            shutil.rmtree(UNINSTALL_REQUEST.parent)
+        sync_parent(UNINSTALL_REQUEST.parent)
+
+    def run(self):
+        if not self.job.get("dataRemoved"):
+            try:
+                self.remove_data()
+            except (AgentError, OSError, ValueError, KeyError) as error:
+                with contextlib.suppress(AgentError):
+                    self.receipt("failed", str(error) if isinstance(error, AgentError) else "Host cleanup could not finish. Check the uninstall service and resolve filesystem errors; cleanup will retry.")
+                raise
+        self.receipt("cleaned")
+        while True:
+            try:
+                self.remove_helper()
+                break
+            except (AgentError, OSError, ValueError):
+                # Once its unit/executable has been unlinked systemd can no
+                # longer restart this worker. Finish idempotently in memory.
+                with contextlib.suppress(AgentError):
+                    self.receipt("failed", "Host files were removed, but removing the uninstall service is still retrying")
+                time.sleep(15)
+        # The controller never marks a host deleted on the earlier receipt.
+        # No credentials or retry state are left on disk at this point.
+        while True:
+            try:
+                self.receipt("completed")
+                return
+            except AgentError:
+                time.sleep(15)
+
+
 class Agent:
     def __init__(self, config, docker=None):
         self.config = config
@@ -300,6 +580,43 @@ class Agent:
         self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
         self.executing_id = None
 
+    def can_uninstall(self):
+        try:
+            if Path(self.config_path) != AGENT_CONFIG:
+                return False
+            uninstall_ownership(self.config)
+            protected_file(UNINSTALL_EXECUTABLE)
+            protected_file(UNINSTALL_UNIT)
+            return True
+        except (AgentError, OSError, ValueError):
+            return False
+
+    def decommission(self, operation):
+        payload = operation.get("payload", {})
+        if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]:
+            raise Rejected("Host removal identity does not match this agent")
+        if payload.get("receiptUrl") != "/api/agent/decommission/" + identifier(operation["id"]) or not isinstance(payload.get("receiptToken"), str) or not payload["receiptToken"]:
+            raise Rejected("Invalid host removal receipt")
+        if not self.can_uninstall():
+            raise Rejected("Reinstall the agent to enable safe removal of its dedicated host files")
+        root = uninstall_ownership(self.config)
+        info = root.stat()
+        protected_file(UNINSTALL_REQUEST, missing=True)
+        aliases_path = root / "agent" / "runtime-images.json"
+        protected_file(aliases_path, missing=True)
+        aliases = json.loads(aliases_path.read_text()) if aliases_path.exists() else {}
+        job = {"hostId": payload["hostId"], "operationId": operation["id"], "receiptUrl": payload["receiptUrl"], "receiptToken": payload["receiptToken"], "controller": self.config["controller"].rstrip("/"), "root": str(root), "rootIdentity": [info.st_dev, info.st_ino], "runtimeImages": aliases}
+        if UNINSTALL_REQUEST.exists():
+            previous = json.loads(UNINSTALL_REQUEST.read_text())
+            if any(previous.get(key) != job[key] for key in ("hostId", "operationId", "receiptUrl", "receiptToken", "controller", "root", "rootIdentity")):
+                raise Rejected("A different host removal is already pending")
+        self.checkpoint({"phase": "uninstall-service"})
+        if not UNINSTALL_REQUEST.exists():
+            atomic_json(UNINSTALL_REQUEST, job)
+        systemctl("enable", "--now", "--no-block", UNINSTALL_UNIT.name)
+        self.stopping.set()
+        return {"decommissioning": True, "message": "The uninstall service will report completion after host files and the agent are removed"}
+
     def instance_dir(self, instance_id, generation):
         instance_id, generation = instance_identity(instance_id, generation)
         return self.root / "instances" / instance_id / str(generation)
@@ -416,7 +733,7 @@ class Agent:
             for line in Path("/etc/os-release").read_text().splitlines():
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
-        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
+        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
         health = self.inspect(instance_id, generation)
@@ -525,6 +842,8 @@ class Agent:
         name = self.container_name(instance_id, generation)
         spec_sha = digest(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode())
         args = ["container", "create", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)]
+        if self.config.get("hostId"):
+            args.extend(["--label", LABEL + ".host=" + identifier(self.config["hostId"])])
         bind_arg = "[" + bind + "]" if ":" in bind else bind
         for protocol in ("udp", "tcp"):
             args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
@@ -923,6 +1242,8 @@ class Agent:
         kind, payload = operation.get("kind"), operation.get("payload", {})
         if not isinstance(kind, str) or not isinstance(payload, dict):
             raise AgentError("Invalid operation")
+        if kind == "host.decommission":
+            return self.decommission(operation)
         if kind == "credentials.rotate":
             pending = "ec_host_" + base64.urlsafe_b64encode(os.urandom(32)).rstrip(b"=").decode()
             self.config["pendingCredential"] = pending
@@ -997,7 +1318,7 @@ class Agent:
             return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message)
         finally:
             self.executing_id = None
-        return self.journal.update(operation["id"], "completed", result)
+        return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result)
 
     def recover_credential(self):
         pending = self.config.get("pendingCredential")
@@ -1024,6 +1345,13 @@ class Agent:
         instance_id, generation = operation.get("instanceId"), operation.get("generation")
         checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
         try:
+            if kind == "host.decommission":
+                self.executing_id = operation["id"]
+                try:
+                    result = self.decommission(operation)
+                    return self.journal.update(operation["id"], "running", result)
+                finally:
+                    self.executing_id = None
             if kind == "credentials.rotate":
                 if not self.config.get("pendingCredential") or self.recover_credential():
                     return self.journal.update(operation["id"], "completed", {"rotated": True, "reconciled": True})
@@ -1183,8 +1511,14 @@ def main():
     parser.add_argument("--token-file")
     parser.add_argument("--root", default="/var/lib/electric-cherry")
     parser.add_argument("--force-enroll", action="store_true")
+    parser.add_argument("--uninstall", metavar="REQUEST", help="Run the installed host removal service")
     parser.add_argument("--version", action="version", version=VERSION)
     args = parser.parse_args()
+    if args.uninstall:
+        if os.geteuid() != 0 or args.enroll:
+            raise AgentError("Host uninstall must run as the dedicated root service")
+        Uninstaller(args.uninstall).run()
+        return
     if args.enroll:
         if not args.controller or not args.token_file:
             parser.error("Enrollment needs --controller and --token-file")