Remove host files and uninstall agent when deleting hosts

agent/ec_agent.py

@@ -29,10 +29,17 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.0.0"
+VERSION = "1.1.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
+AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
+AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
+AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service")
+UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
+UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service")
+UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
+ROOT_MARKER = ".electric-cherry.json"
 
 
 class AgentError(Exception):
@@ -47,6 +54,97 @@ class Rejected(AgentError):
     """A definitive precondition failure, before any destructive action."""
 
 
+def systemctl(*args):
+    try:
+        result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
+    except (OSError, subprocess.TimeoutExpired):
+        raise AgentError("Host service management is unavailable") from None
+    if result.returncode:
+        raise AgentError("Host service management did not complete")
+
+
+def protected_path(path, missing=False):
+    """Deletion authority must never come from a runtime-writable path."""
+    path = Path(path)
+    for current in (*reversed(path.parents), path):
+        try:
+            info = current.lstat()
+        except FileNotFoundError:
+            if missing:
+                continue
+            raise AgentError("Host uninstall files are unavailable; reinstall the agent") from None
+        if stat_module.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+            raise AgentError("Host uninstall paths must be owned and writable only by root")
+    return path
+
+
+def protected_file(path, missing=False):
+    path = protected_path(path, missing=missing)
+    try:
+        if not stat_module.S_ISREG(path.lstat().st_mode):
+            raise AgentError("Host uninstall metadata must be a regular file")
+    except FileNotFoundError:
+        if not missing:
+            raise
+    return path
+
+
+def sync_parent(path):
+    descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def uninstall_root(value, identity=None):
+    """Accept only a canonical, dedicated directory with no nested mounts."""
+    if not isinstance(value, str) or not re.fullmatch(r"/[A-Za-z0-9/_.-]+", value):
+        raise AgentError("Host data directory is unsafe to remove")
+    root = Path(value)
+    forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"}
+    if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden:
+        raise AgentError("Host data directory is unsafe to remove")
+    for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT):
+        if installed == root or root in installed.parents:
+            raise AgentError("Host data directory overlaps the agent installation")
+    protected_path(root, missing=True)
+    if root.exists():
+        info = root.stat()
+        if not stat_module.S_ISDIR(info.st_mode):
+            raise AgentError("Host data directory is not a directory")
+        if identity and [info.st_dev, info.st_ino] != identity:
+            raise AgentError("Host data directory changed since deletion was authorized")
+    try:
+        # ProtectSystem/ReadWritePaths give the running agent a private bind
+        # mount at its data directory. Inspect the host systemd namespace,
+        # otherwise every legitimate installation appears to be a mount root.
+        mount_lines = Path("/proc/1/mountinfo").read_text().splitlines()
+    except FileNotFoundError:
+        mount_lines = []  # Non-Linux tests; the installed service requires Linux.
+    for line in mount_lines:
+        fields = line.split()
+        if len(fields) < 5:
+            continue
+        mount = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), fields[4])
+        if mount == value or mount.startswith(value + "/"):
+            raise AgentError("Unmount filesystems inside the host data directory before deletion")
+    return root
+
+
+def uninstall_ownership(config):
+    root = uninstall_root(config.get("root"))
+    marker = protected_file(root / ROOT_MARKER)
+    try:
+        saved = json.loads(marker.read_text())
+    except (OSError, ValueError):
+        raise AgentError("Host data ownership is unavailable; reinstall the agent") from None
+    expected = {"version": 1, "root": str(root), "hostId": config.get("hostId"), "controller": config.get("controller", "").rstrip("/")}
+    if saved != expected or not expected["hostId"]:
+        raise AgentError("Host data ownership does not match this enrollment")
+    return root
+
+
 def digest(data):
     return hashlib.sha256(data).hexdigest()
 
@@ -275,7 +373,7 @@ class Docker:
             raise AgentError("Docker command unavailable or timed out") from None
         if result.returncode:
             # Never include arbitrary Docker stderr: create errors may echo secrets.
-            if tolerate_missing and (b"No such object" in (result.stderr or b"") or b"No such container" in (result.stderr or b"")):
+            if tolerate_missing and any(message in (result.stderr or b"") for message in (b"No such object", b"No such container", b"No such image")):
                 return None
             raise AgentError("Docker rejected the requested operation")
         return result.stdout.decode("utf-8", "replace")
@@ -285,6 +383,188 @@ class Docker:
         return json.loads(raw)[0] if raw is not None else None
 
 
+class Uninstaller:
+    """A separate systemd service owns cleanup after stopping the host agent.
+
+    Its root-only request and executable stay on disk until the controller
+    acknowledges data removal. Completion is reported only after removing those
+    final files. A lost final response can be retried from memory; a power loss
+    in that last window deliberately leaves controller status unconfirmed.
+    """
+
+    def __init__(self, request_path, docker=None):
+        if Path(request_path) != UNINSTALL_REQUEST:
+            raise AgentError("Invalid host uninstall request location")
+        protected_file(UNINSTALL_REQUEST)
+        self.job = json.loads(UNINSTALL_REQUEST.read_text())
+        self.docker = docker or Docker()
+        self.root = uninstall_root(self.job.get("root"), self.job.get("rootIdentity"))
+        host_id = identifier(self.job.get("hostId"))
+        operation_id = identifier(self.job.get("operationId"))
+        parsed = urllib.parse.urlsplit(self.job.get("controller", ""))
+        if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise AgentError("Host uninstall controller must use HTTPS")
+        if self.job.get("receiptUrl") != "/api/agent/decommission/" + operation_id or not isinstance(self.job.get("receiptToken"), str) or not self.job["receiptToken"]:
+            raise AgentError("Invalid host uninstall receipt")
+        if not isinstance(self.job.get("rootIdentity"), list) or len(self.job["rootIdentity"]) != 2 or not all(isinstance(value, int) and not isinstance(value, bool) for value in self.job["rootIdentity"]):
+            raise AgentError("Invalid host data ownership proof")
+        if self.root.exists():
+            # The authorized device/inode is persisted before deletion so a
+            # crash halfway through rmtree does not require a deleted marker.
+            if not self.job.get("removalStarted"):
+                uninstall_ownership({"root": str(self.root), "controller": self.job["controller"], "hostId": host_id})
+
+    def save(self, **changes):
+        self.job.update(changes)
+        atomic_json(UNINSTALL_REQUEST, self.job)
+
+    def receipt(self, stage, error=None):
+        payload = {"stage": stage}
+        if error:
+            payload["error"] = error
+        request = urllib.request.Request(self.job["controller"].rstrip("/") + self.job["receiptUrl"], data=json.dumps(payload).encode(), headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.job["receiptToken"], "User-Agent": "electric-cherry-agent/" + VERSION}, method="POST")
+        class NoRedirect(urllib.request.HTTPRedirectHandler):
+            def redirect_request(self, req, fp, code, msg, hdrs, newurl):
+                raise AgentError("Controller redirects are not allowed")
+        try:
+            with urllib.request.build_opener(NoRedirect()).open(request, timeout=35) as response:
+                if json.load(response).get("acknowledged") is not True:
+                    raise AgentError("Controller has not acknowledged host removal")
+        except (OSError, ValueError):
+            raise AgentError("Host removal receipt is waiting for controller contact") from None
+
+    def containers(self):
+        ids = self.docker.call("container", "ls", "--all", "--quiet", "--no-trunc").splitlines()
+        values = []
+        for container_id in ids:
+            info = self.docker.inspect(container_id)
+            if info:
+                values.append(info)
+        return values
+
+    def owned_container(self, info):
+        labels = info.get("Config", {}).get("Labels", {}) or {}
+        if labels.get(LABEL + ".managed") != "true":
+            return False
+        if labels.get(LABEL + ".host") not in (None, self.job["hostId"]):
+            return False
+        try:
+            instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", "")))
+        except (AgentError, ValueError, TypeError):
+            return False
+        expected = str(self.root / "instances" / instance_id / str(generation) / "app")
+        # Legacy agents did not label the host; its exact managed bind mount
+        # proves ownership without touching another installation's containers.
+        return any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", []))
+
+    def uses_root(self, info):
+        for mount in info.get("Mounts", []):
+            if mount.get("Type") != "bind":
+                continue
+            source = Path(mount.get("Source", "/")).resolve()
+            if source == self.root or self.root in source.parents:
+                return True
+        return False
+
+    def remove_docker_resources(self):
+        containers = self.containers()
+        if any(self.uses_root(info) and not self.owned_container(info) for info in containers):
+            raise AgentError("An unrelated container uses the host data directory; detach it before retrying deletion")
+        owned = [info for info in containers if self.owned_container(info)]
+        if any(info.get("State", {}).get("Running") is not False or info.get("State", {}).get("Status") not in ("created", "exited", "dead") for info in owned):
+            raise AgentError("A managed game container is still running or its state is unknown; retire it before deleting this host")
+        for info in owned:
+            # Do not force removal: Docker must also reject a container that
+            # starts again after inspection rather than interrupting players.
+            self.docker.call("container", "rm", "--volumes", info["Id"], timeout=120, tolerate_missing=True)
+        remaining = self.containers()
+        if any(self.uses_root(info) for info in remaining):
+            raise AgentError("Containers still use the host data directory")
+        # Never prune Docker globally. Only remove this agent's recorded image
+        # references, and preserve images used by any unrelated container.
+        used_images = {info.get("Image") for info in remaining}
+        images = self.job.get("runtimeImages", {})
+        if not isinstance(images, dict):
+            raise AgentError("Managed runtime image ownership is unavailable")
+        for reference, expected_id in images.items():
+            if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,511}", reference) or not isinstance(expected_id, str):
+                raise AgentError("Managed runtime image ownership is invalid")
+            raw = self.docker.call("image", "inspect", reference, tolerate_missing=True)
+            if raw is None:
+                continue
+            found = json.loads(raw)
+            if not found or found[0].get("Id") != expected_id or expected_id in used_images:
+                continue
+            self.docker.call("image", "rm", reference, tolerate_missing=True)
+
+    def remove_data(self):
+        protected_path(AGENT_CONFIG.parent, missing=True)
+        if not self.job.get("agentStopped"):
+            systemctl("disable", "--now", AGENT_UNIT.name)
+            self.save(agentStopped=True)
+        self.remove_docker_resources()
+        root = uninstall_root(str(self.root), self.job["rootIdentity"])
+        # This checkpoint also authorizes safe continuation if a power loss
+        # removes the ownership marker before the rest of the tree.
+        self.save(removalStarted=True)
+        if root.exists():
+            if not shutil.rmtree.avoids_symlink_attacks:
+                raise AgentError("This platform cannot safely remove managed host files")
+            shutil.rmtree(root)
+            sync_parent(root)
+        for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT):
+            protected_file(path, missing=True)
+            path.unlink(missing_ok=True)
+            sync_parent(path)
+        systemctl("daemon-reload")
+        self.save(dataRemoved=True)
+
+    def remove_helper(self):
+        # Disable does not stop this process: it must outlive its own unit and
+        # executable to deliver the final receipt after the files are gone.
+        protected_file(UNINSTALL_UNIT, missing=True)
+        if UNINSTALL_UNIT.exists():
+            systemctl("disable", UNINSTALL_UNIT.name)
+            UNINSTALL_UNIT.unlink()
+        sync_parent(UNINSTALL_UNIT)
+        systemctl("daemon-reload")
+        protected_file(UNINSTALL_EXECUTABLE, missing=True)
+        UNINSTALL_EXECUTABLE.unlink(missing_ok=True)
+        sync_parent(UNINSTALL_EXECUTABLE)
+        protected_path(UNINSTALL_REQUEST.parent, missing=True)
+        if UNINSTALL_REQUEST.parent.exists():
+            shutil.rmtree(UNINSTALL_REQUEST.parent)
+        sync_parent(UNINSTALL_REQUEST.parent)
+
+    def run(self):
+        if not self.job.get("dataRemoved"):
+            try:
+                self.remove_data()
+            except (AgentError, OSError, ValueError, KeyError) as error:
+                with contextlib.suppress(AgentError):
+                    self.receipt("failed", str(error) if isinstance(error, AgentError) else "Host cleanup could not finish. Check the uninstall service and resolve filesystem errors; cleanup will retry.")
+                raise
+        self.receipt("cleaned")
+        while True:
+            try:
+                self.remove_helper()
+                break
+            except (AgentError, OSError, ValueError):
+                # Once its unit/executable has been unlinked systemd can no
+                # longer restart this worker. Finish idempotently in memory.
+                with contextlib.suppress(AgentError):
+                    self.receipt("failed", "Host files were removed, but removing the uninstall service is still retrying")
+                time.sleep(15)
+        # The controller never marks a host deleted on the earlier receipt.
+        # No credentials or retry state are left on disk at this point.
+        while True:
+            try:
+                self.receipt("completed")
+                return
+            except AgentError:
+                time.sleep(15)
+
+
 class Agent:
     def __init__(self, config, docker=None):
         self.config = config
@@ -300,6 +580,43 @@ class Agent:
         self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
         self.executing_id = None
 
+    def can_uninstall(self):
+        try:
+            if Path(self.config_path) != AGENT_CONFIG:
+                return False
+            uninstall_ownership(self.config)
+            protected_file(UNINSTALL_EXECUTABLE)
+            protected_file(UNINSTALL_UNIT)
+            return True
+        except (AgentError, OSError, ValueError):
+            return False
+
+    def decommission(self, operation):
+        payload = operation.get("payload", {})
+        if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]:
+            raise Rejected("Host removal identity does not match this agent")
+        if payload.get("receiptUrl") != "/api/agent/decommission/" + identifier(operation["id"]) or not isinstance(payload.get("receiptToken"), str) or not payload["receiptToken"]:
+            raise Rejected("Invalid host removal receipt")
+        if not self.can_uninstall():
+            raise Rejected("Reinstall the agent to enable safe removal of its dedicated host files")
+        root = uninstall_ownership(self.config)
+        info = root.stat()
+        protected_file(UNINSTALL_REQUEST, missing=True)
+        aliases_path = root / "agent" / "runtime-images.json"
+        protected_file(aliases_path, missing=True)
+        aliases = json.loads(aliases_path.read_text()) if aliases_path.exists() else {}
+        job = {"hostId": payload["hostId"], "operationId": operation["id"], "receiptUrl": payload["receiptUrl"], "receiptToken": payload["receiptToken"], "controller": self.config["controller"].rstrip("/"), "root": str(root), "rootIdentity": [info.st_dev, info.st_ino], "runtimeImages": aliases}
+        if UNINSTALL_REQUEST.exists():
+            previous = json.loads(UNINSTALL_REQUEST.read_text())
+            if any(previous.get(key) != job[key] for key in ("hostId", "operationId", "receiptUrl", "receiptToken", "controller", "root", "rootIdentity")):
+                raise Rejected("A different host removal is already pending")
+        self.checkpoint({"phase": "uninstall-service"})
+        if not UNINSTALL_REQUEST.exists():
+            atomic_json(UNINSTALL_REQUEST, job)
+        systemctl("enable", "--now", "--no-block", UNINSTALL_UNIT.name)
+        self.stopping.set()
+        return {"decommissioning": True, "message": "The uninstall service will report completion after host files and the agent are removed"}
+
     def instance_dir(self, instance_id, generation):
         instance_id, generation = instance_identity(instance_id, generation)
         return self.root / "instances" / instance_id / str(generation)
@@ -416,7 +733,7 @@ class Agent:
             for line in Path("/etc/os-release").read_text().splitlines():
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
-        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
+        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
         health = self.inspect(instance_id, generation)
@@ -525,6 +842,8 @@ class Agent:
         name = self.container_name(instance_id, generation)
         spec_sha = digest(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode())
         args = ["container", "create", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)]
+        if self.config.get("hostId"):
+            args.extend(["--label", LABEL + ".host=" + identifier(self.config["hostId"])])
         bind_arg = "[" + bind + "]" if ":" in bind else bind
         for protocol in ("udp", "tcp"):
             args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
@@ -923,6 +1242,8 @@ class Agent:
         kind, payload = operation.get("kind"), operation.get("payload", {})
         if not isinstance(kind, str) or not isinstance(payload, dict):
             raise AgentError("Invalid operation")
+        if kind == "host.decommission":
+            return self.decommission(operation)
         if kind == "credentials.rotate":
             pending = "ec_host_" + base64.urlsafe_b64encode(os.urandom(32)).rstrip(b"=").decode()
             self.config["pendingCredential"] = pending
@@ -997,7 +1318,7 @@ class Agent:
             return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message)
         finally:
             self.executing_id = None
-        return self.journal.update(operation["id"], "completed", result)
+        return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result)
 
     def recover_credential(self):
         pending = self.config.get("pendingCredential")
@@ -1024,6 +1345,13 @@ class Agent:
         instance_id, generation = operation.get("instanceId"), operation.get("generation")
         checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
         try:
+            if kind == "host.decommission":
+                self.executing_id = operation["id"]
+                try:
+                    result = self.decommission(operation)
+                    return self.journal.update(operation["id"], "running", result)
+                finally:
+                    self.executing_id = None
             if kind == "credentials.rotate":
                 if not self.config.get("pendingCredential") or self.recover_credential():
                     return self.journal.update(operation["id"], "completed", {"rotated": True, "reconciled": True})
@@ -1183,8 +1511,14 @@ def main():
     parser.add_argument("--token-file")
     parser.add_argument("--root", default="/var/lib/electric-cherry")
     parser.add_argument("--force-enroll", action="store_true")
+    parser.add_argument("--uninstall", metavar="REQUEST", help="Run the installed host removal service")
     parser.add_argument("--version", action="version", version=VERSION)
     args = parser.parse_args()
+    if args.uninstall:
+        if os.geteuid() != 0 or args.enroll:
+            raise AgentError("Host uninstall must run as the dedicated root service")
+        Uninstaller(args.uninstall).run()
+        return
     if args.enroll:
         if not args.controller or not args.token_file:
             parser.error("Enrollment needs --controller and --token-file")

agent/install.sh

@@ -21,7 +21,7 @@ done
 case "$controller" in https://*) ;; *) echo '--controller must be an HTTPS URL.' >&2; exit 2 ;; esac
 [ -n "$enrollment_token" ] || { echo '--token is required.' >&2; exit 2; }
 case "$data_dir" in /*) ;; *) echo '--data-dir must be absolute.' >&2; exit 2 ;; esac
-case "$data_dir" in *[!a-zA-Z0-9/_.-]*|/|/etc|/usr|/var|/var/lib) echo 'Choose a dedicated data directory.' >&2; exit 2 ;; esac
+case "$data_dir" in *[!a-zA-Z0-9/_.-]*|*/../*|*/./*|*/..|*/.|*//*|*/) echo 'Choose a canonical, dedicated data directory.' >&2; exit 2 ;; esac
 [ ! -L "$data_dir" ] || { echo 'Data directory must not be a symlink.' >&2; exit 2; }
 . /etc/os-release
 [ "${ID:-}" = debian ] || { echo 'Supported platform: Debian 12 or 13.' >&2; exit 1; }
@@ -32,13 +32,82 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; }
 export DEBIAN_FRONTEND=noninteractive
 apt-get update -qq
 apt-get install -y --no-install-recommends ca-certificates curl python3
-python3 - "$controller" "$data_dir" <<'PY'
-import json, pathlib, sys
-config = pathlib.Path('/etc/electric-cherry/agent.json')
-if config.exists():
-    saved = json.loads(config.read_text())
-    if saved.get('controller', '').rstrip('/') != sys.argv[1].rstrip('/') or pathlib.Path(saved.get('root', '')).resolve() != pathlib.Path(sys.argv[2]).resolve():
+python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY'
+# INSTALLATION_PREFLIGHT: this runs before touching managed directories or services.
+import json, pathlib, re, stat, sys
+
+controller, raw_root, config_name = sys.argv[1:]
+root = pathlib.Path(raw_root)
+config = pathlib.Path(config_name)
+reserved = {
+    '/', '/bin', '/boot', '/dev', '/etc', '/home', '/lib', '/lib64',
+    '/media', '/mnt', '/opt', '/proc', '/root', '/run', '/sbin', '/srv',
+    '/sys', '/tmp', '/usr', '/usr/local', '/usr/local/bin', '/usr/local/lib',
+    '/usr/local/libexec', '/var', '/var/cache', '/var/lib', '/var/log', '/var/tmp',
+}
+if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root)
+        or str(root) != raw_root or any(part in ('.', '..') for part in raw_root.split('/')[1:])
+        or len(root.parts) < 3 or raw_root in reserved or config.parent == root or config.parent in root.parents):
+    raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.')
+installed_paths = [config,
+    pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service')]
+if any(root == path or root in path.parents for path in installed_paths):
+    raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.')
+
+def trusted_directories(path):
+    for current in [*reversed(path.parents), path]:
+        try:
+            info = current.lstat()
+        except FileNotFoundError:
+            continue
+        if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+            raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.')
+
+def trusted_json(path):
+    info = path.lstat()
+    if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+        raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.')
+    value = json.loads(path.read_text())
+    if not isinstance(value, dict):
+        raise SystemExit('Existing installation metadata is invalid.')
+    return value
+
+trusted_directories(root)
+trusted_directories(config.parent)
+mountinfo = pathlib.Path('/proc/1/mountinfo')
+if mountinfo.exists():
+    for line in mountinfo.read_text().splitlines():
+        fields = line.split()
+        if len(fields) < 5:
+            continue
+        mount = re.sub(r'\\([0-7]{3})', lambda match: chr(int(match[1], 8)), fields[4])
+        if mount == raw_root or mount.startswith(raw_root + '/'):
+            raise SystemExit('Use a dedicated subdirectory of the mounted disk, with no filesystems mounted inside it, so host deletion can remove the entire data directory.')
+request = config.with_name('decommission.json')
+if request.exists() or request.is_symlink():
+    raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.')
+saved = None
+if config.exists() or config.is_symlink():
+    saved = trusted_json(config)
+    if saved.get('controller', '').rstrip('/') != controller.rstrip('/') or saved.get('root') != raw_root:
         raise SystemExit('Existing enrollment uses a different controller or data directory. Preserve the existing installation and migrate it explicitly.')
+    if not saved.get('hostId') or not saved.get('credential'):
+        raise SystemExit('Existing enrollment is incomplete. Repair its credentials before upgrading.')
+marker = root / '.electric-cherry.json'
+if marker.exists() or marker.is_symlink():
+    owned = trusted_json(marker)
+    if not saved or owned != {'version': 1, 'root': raw_root, 'hostId': saved['hostId'], 'controller': controller.rstrip('/')}:
+        raise SystemExit('Data directory ownership does not match this enrollment. Preserve it and migrate explicitly.')
+elif root.exists() and any(root.iterdir()):
+    # Upgrade installations predating ownership markers only when the entire
+    # root has the known layout. Never adopt a directory of unrelated files.
+    if not saved or any(path.name not in {'agent', 'instances', 'libraries'} for path in root.iterdir()):
+        raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.')
+for directory in ('agent', 'instances', 'libraries'):
+    trusted_directories(root / directory)
 PY
 if ! command -v docker >/dev/null; then
     if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then
@@ -76,9 +145,36 @@ install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-
     install -d -m 0755 /usr/local/libexec
     install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py
 }
+# A separate worker survives stopping/removing the agent service. Its only
+# entry point is a fixed, authenticated deletion request prepared by the agent.
+install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py
 printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token"
 unset enrollment_token
 python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"}
+python3 - "$data_dir" /etc/electric-cherry/agent.json <<'PY'
+# INSTALLATION_OWNERSHIP: bind deletion authority to this exact enrollment/root.
+import json, os, pathlib, sys, uuid
+root = pathlib.Path(sys.argv[1])
+config = json.loads(pathlib.Path(sys.argv[2]).read_text())
+if config.get('root') != str(root) or not config.get('hostId') or not config.get('controller'):
+    raise SystemExit('Enrollment did not establish ownership of this data directory.')
+marker = root / '.electric-cherry.json'
+temporary = root / ('.electric-cherry.' + uuid.uuid4().hex + '.tmp')
+try:
+    with temporary.open('x') as stream:
+        os.chmod(temporary, 0o600)
+        json.dump({'version': 1, 'root': str(root), 'hostId': config['hostId'], 'controller': config['controller'].rstrip('/')}, stream)
+        stream.flush()
+        os.fsync(stream.fileno())
+    os.replace(temporary, marker)
+    directory_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(directory_fd)
+    finally:
+        os.close(directory_fd)
+finally:
+    temporary.unlink(missing_ok=True)
+PY
 cat > /etc/systemd/system/electric-cherry-agent.service <<EOF
 [Unit]
 Description=Electric Cherry Host Agent
@@ -99,6 +195,23 @@ ProtectHome=true
 PrivateTmp=true
 ReadWritePaths=$data_dir /etc/electric-cherry
 
+[Install]
+WantedBy=multi-user.target
+EOF
+cat > /etc/systemd/system/electric-cherry-uninstall.service <<'EOF'
+[Unit]
+Description=Remove an Electric Cherry Host Installation
+After=network-online.target docker.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-uninstall.py --uninstall /etc/electric-cherry/decommission.json
+Restart=on-failure
+RestartSec=15
+TimeoutStartSec=0
+UMask=0077
+
 [Install]
 WantedBy=multi-user.target
 EOF

agent/package_release.py

@@ -10,4 +10,4 @@ target = source.parent / "static" / "agent"
 target.mkdir(parents=True, exist_ok=True)
 shutil.copyfile(source / "ec_agent.py", target / "ec-agent.py")
 shutil.copyfile(source / "install.sh", target / "install.sh")
-(target / "manifest.json").write_text(json.dumps({"version": "1.0.0", "protocolVersion": 1, "platform": "debian-amd64", "sha256": hashlib.sha256((target / "ec-agent.py").read_bytes()).hexdigest()}, indent=2) + "\n")
+(target / "manifest.json").write_text(json.dumps({"version": "1.1.0", "protocolVersion": 1, "platform": "debian-amd64", "sha256": hashlib.sha256((target / "ec-agent.py").read_bytes()).hexdigest()}, indent=2) + "\n")

agent/tests/test_installer.py

@@ -0,0 +1,185 @@
+"""Exercise installer guards without installing software or touching services."""
+
+import json
+import os
+from pathlib import Path
+import re
+import stat
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+
+SOURCE = (Path(__file__).resolve().parents[1] / "install.sh").read_text()
+BLOCKS = re.findall(r"<<'PY'\n(.*?)\nPY", SOURCE, re.DOTALL)
+PREFLIGHT = next(block for block in BLOCKS if "# INSTALLATION_PREFLIGHT:" in block)
+OWNERSHIP = next(block for block in BLOCKS if "# INSTALLATION_OWNERSHIP:" in block)
+
+
+class InstallerTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.base = Path(self.temporary.name).resolve()
+        self.root = self.base / "managed"
+        self.config = self.base / "configuration" / "agent.json"
+        self.config.parent.mkdir()
+        self.controller = "https://controller.example"
+        self.saved = {"root": str(self.root), "controller": self.controller,
+                      "hostId": "host-one", "credential": "private"}
+        self.overrides = {}
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def preflight(self, root=None):
+        original_lstat = Path.lstat
+
+        def root_owned(path):
+            # Simulate Linux root-owned parents while keeping the actual inode
+            # types. No filesystem mutations or elevated privileges are needed.
+            info = original_lstat(path)
+            fields = list(info)
+            fields[4] = 0
+            if stat.S_ISDIR(info.st_mode):
+                fields[0] &= ~0o022
+            override = self.overrides.get(str(path), {})
+            fields[0] = override.get("mode", fields[0])
+            fields[4] = override.get("uid", fields[4])
+            return os.stat_result(fields)
+
+        argv = ["-", self.controller, str(root or self.root), str(self.config)]
+        with patch.object(sys, "argv", argv), patch.object(Path, "lstat", root_owned):
+            exec(compile(PREFLIGHT, "installer-preflight", "exec"), {})
+
+    def legacy_installation(self):
+        for directory in ("agent", "instances", "libraries"):
+            (self.root / directory).mkdir(parents=True, exist_ok=True)
+        self.config.write_text(json.dumps(self.saved))
+
+    def ownership(self):
+        with patch.object(sys, "argv", ["-", str(self.root), str(self.config)]):
+            exec(compile(OWNERSHIP, "installer-ownership", "exec"), {})
+
+    def test_new_directory_and_empty_directory_are_allowed_without_adopting_files(self):
+        self.preflight()
+        self.assertFalse(self.root.exists())
+        self.root.mkdir()
+        self.preflight()
+        (self.root / "unrelated.txt").write_text("keep")
+        with self.assertRaisesRegex(SystemExit, "not owned"):
+            self.preflight()
+        self.assertEqual((self.root / "unrelated.txt").read_text(), "keep")
+
+    def test_known_legacy_layout_can_upgrade_and_marker_binds_exact_enrollment(self):
+        self.legacy_installation()
+        original_config = self.config.read_bytes()
+        (self.root / "libraries" / "game.ff").write_bytes(b"game data")
+        self.preflight()
+        self.ownership()
+        marker = self.root / ".electric-cherry.json"
+        self.assertEqual(json.loads(marker.read_text()), {
+            "version": 1, "root": str(self.root), "hostId": "host-one", "controller": self.controller})
+        self.assertEqual(stat.S_IMODE(marker.stat().st_mode), 0o600)
+        self.assertEqual(self.config.read_bytes(), original_config)
+        self.assertEqual((self.root / "libraries" / "game.ff").read_bytes(), b"game data")
+        self.preflight()
+
+    def test_legacy_upgrade_never_adopts_unrelated_top_level_files(self):
+        self.legacy_installation()
+        (self.root / "personal-backup.tar").write_bytes(b"keep")
+        with self.assertRaisesRegex(SystemExit, "not owned"):
+            self.preflight()
+        self.assertFalse((self.root / ".electric-cherry.json").exists())
+
+    def test_reenrollment_cannot_silently_change_controller_or_root(self):
+        self.legacy_installation()
+        for field, value in (("controller", "https://another.example"), ("root", str(self.base / "other"))):
+            with self.subTest(field=field):
+                self.config.write_text(json.dumps({**self.saved, field: value}))
+                with self.assertRaisesRegex(SystemExit, "different controller or data directory"):
+                    self.preflight()
+
+    def test_existing_marker_must_match_controller_host_and_root(self):
+        self.legacy_installation()
+        self.ownership()
+        marker = self.root / ".electric-cherry.json"
+        valid = json.loads(marker.read_text())
+        for field, value in (("hostId", "host-two"), ("root", str(self.base)), ("controller", "https://another.example")):
+            with self.subTest(field=field):
+                marker.write_text(json.dumps({**valid, field: value}))
+                with self.assertRaisesRegex(SystemExit, "ownership does not match"):
+                    self.preflight()
+
+    def test_lexical_aliases_and_system_roots_are_rejected_before_mutation(self):
+        for root in ("/var/lib/..", "/var/lib/../lib", "/var//lib/managed", "/var/lib/./managed", "/var/lib/managed/", "/var/lib", "/usr/local/libexec", "/", "/games", str(self.config.parent), str(self.config.parent / "data")):
+            with self.subTest(root=root), self.assertRaisesRegex(SystemExit, "canonical, dedicated"):
+                self.preflight(root)
+        self.assertFalse(self.root.exists())
+
+    def test_symlink_roots_ancestors_metadata_and_managed_subdirectories_are_rejected(self):
+        elsewhere = self.base / "elsewhere"
+        elsewhere.mkdir()
+        self.root.symlink_to(elsewhere, target_is_directory=True)
+        with self.assertRaisesRegex(SystemExit, "not symlinks"):
+            self.preflight()
+        with self.assertRaisesRegex(SystemExit, "not symlinks"):
+            self.preflight(str(self.root / "nested"))
+        self.root.unlink()
+        self.legacy_installation()
+        (self.root / "libraries").rmdir()
+        (self.root / "libraries").symlink_to(elsewhere, target_is_directory=True)
+        with self.assertRaisesRegex(SystemExit, "not symlinks"):
+            self.preflight()
+        (self.root / "libraries").unlink()
+        (self.root / "libraries").mkdir()
+        self.config.unlink()
+        real_config = self.base / "real-config.json"
+        real_config.write_text(json.dumps(self.saved))
+        self.config.symlink_to(real_config)
+        with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
+            self.preflight()
+
+    def test_untrusted_parents_and_metadata_cannot_authorize_cleanup(self):
+        self.legacy_installation()
+        self.overrides[str(self.base)] = {"uid": 1000}
+        with self.assertRaisesRegex(SystemExit, "root-owned"):
+            self.preflight()
+        self.overrides = {str(self.base): {"mode": stat.S_IFDIR | 0o777}}
+        with self.assertRaisesRegex(SystemExit, "root-owned"):
+            self.preflight()
+        self.overrides = {str(self.config): {"uid": 1000}}
+        with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
+            self.preflight()
+
+    def test_pending_deletion_blocks_upgrade_even_with_dangling_request_symlink(self):
+        self.legacy_installation()
+        request = self.config.with_name("decommission.json")
+        request.write_text("{}")
+        with self.assertRaisesRegex(SystemExit, "deletion is in progress"):
+            self.preflight()
+        request.unlink()
+        request.symlink_to(self.base / "missing")
+        with self.assertRaisesRegex(SystemExit, "deletion is in progress"):
+            self.preflight()
+
+    def test_mounted_parent_is_allowed_but_data_root_and_nested_mounts_are_rejected(self):
+        mountinfo = Path('/proc/1/mountinfo')
+        original_exists, original_read = Path.exists, Path.read_text
+        for mount, allowed in ((self.base, True), (self.root, False), (self.root / 'libraries', False)):
+            def exists(path):
+                return True if path == mountinfo else original_exists(path)
+
+            def read(path, *args, **kwargs):
+                return f'29 24 8:1 / {mount} rw - ext4 /dev/sda1 rw' if path == mountinfo else original_read(path, *args, **kwargs)
+
+            with self.subTest(mount=mount), patch.object(Path, 'exists', exists), patch.object(Path, 'read_text', read):
+                if allowed:
+                    self.preflight()
+                else:
+                    with self.assertRaisesRegex(SystemExit, "dedicated subdirectory of the mounted disk"):
+                        self.preflight()
+
+
+if __name__ == "__main__":
+    unittest.main()

agent/tests/test_uninstall.py

@@ -0,0 +1,338 @@
+import importlib.util
+import json
+import os
+from pathlib import Path
+import shutil
+import stat
+import tempfile
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("ec_uninstall_agent", Path(__file__).resolve().parents[1] / "ec_agent.py")
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+INSTANCE = "a53d99da-a1eb-4f01-9b10-a6268b556542"
+
+
+class CleanupDocker:
+    def __init__(self):
+        self.calls = []
+        self.items = {}
+        self.images = {}
+
+    def inspect(self, identity):
+        return self.items.get(identity)
+
+    def call(self, *args, **kwargs):
+        self.calls.append(args)
+        if args[:2] == ("container", "ls"):
+            return "\n".join(self.items)
+        if args[:2] == ("container", "rm"):
+            self.items.pop(args[-1], None)
+            return ""
+        if args[:2] == ("image", "inspect"):
+            item = self.images.get(args[-1])
+            return json.dumps([item]) if item else None
+        if args[:2] == ("image", "rm"):
+            self.images.pop(args[-1], None)
+            return ""
+        raise AssertionError("Unexpected Docker request: " + str(args))
+
+
+class UninstallTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.base = Path(self.temp.name).resolve()
+        self.root = self.base / "host-data"
+        self.config_dir = self.base / "etc" / "electric-cherry"
+        self.bin_dir = self.base / "libexec"
+        self.unit_dir = self.base / "systemd"
+        for directory in (self.config_dir, self.bin_dir, self.unit_dir):
+            directory.mkdir(parents=True)
+        replacements = {
+            "AGENT_CONFIG": self.config_dir / "agent.json",
+            "AGENT_EXECUTABLE": self.bin_dir / "electric-cherry-agent.py",
+            "AGENT_UNIT": self.unit_dir / "electric-cherry-agent.service",
+            "UNINSTALL_EXECUTABLE": self.bin_dir / "electric-cherry-uninstall.py",
+            "UNINSTALL_UNIT": self.unit_dir / "electric-cherry-uninstall.service",
+            "UNINSTALL_REQUEST": self.config_dir / "decommission.json",
+        }
+        self.patches = [patch.object(module, key, value) for key, value in replacements.items()]
+        # Unit tests run without root privileges. Keep actual path, symlink,
+        # file-type and containment checks while substituting owner checks.
+        def test_protected_path(value, missing=False):
+            path = Path(value)
+            if any(current.is_symlink() for current in (path, *path.parents)):
+                raise module.AgentError("Symlink ownership is invalid")
+            if not missing and not path.exists():
+                raise module.AgentError("Host uninstall files are unavailable")
+            return path
+        self.patches.extend([patch.object(module, "protected_path", side_effect=test_protected_path), patch.object(module, "systemctl"), patch.object(module, "sync_parent")])
+        for entry in self.patches:
+            entry.start()
+        self.config = {"hostId": "host-one", "controller": "https://controller.example", "credential": "secret", "root": str(self.root), "configPath": str(module.AGENT_CONFIG)}
+        self.agent = module.Agent(self.config)
+        self.docker = CleanupDocker()
+        for path in (module.AGENT_EXECUTABLE, module.AGENT_UNIT, module.UNINSTALL_EXECUTABLE, module.UNINSTALL_UNIT):
+            path.write_text("installed")
+        module.atomic_json(module.AGENT_CONFIG, self.config)
+        module.atomic_json(self.root / module.ROOT_MARKER, {"version": 1, "root": str(self.root), "hostId": "host-one", "controller": "https://controller.example"})
+        self.game = self.root / "libraries" / "t6" / "library" / "content" / "zone" / "all" / "dlc.ff"
+        self.game.parent.mkdir(parents=True)
+        self.game.write_bytes(b"game DLC")
+        (self.base / "unrelated.txt").write_text("keep")
+        self.operation = {"id": "delete-one", "kind": "host.decommission", "payload": {"hostId": "host-one", "operationId": "delete-one", "receiptToken": "receipt-only-secret", "receiptUrl": "/api/agent/decommission/delete-one"}}
+
+    def tearDown(self):
+        self.agent.journal.close()
+        for entry in reversed(self.patches):
+            entry.stop()
+        self.temp.cleanup()
+
+    def prepare(self):
+        result = self.agent.execute(self.operation)
+        self.assertEqual(result["state"], "running", result)
+        return module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
+
+    def container(self, identity, owned=True, host="host-one", root=None, image="image-one"):
+        labels = {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1", module.LABEL + ".host": host} if owned else {}
+        return {"Id": identity, "Image": image, "Config": {"Labels": labels}, "State": {"Running": False, "Status": "exited"}, "Mounts": [{"Type": "bind", "Source": str((root or self.root) / "instances" / INSTANCE / "1" / "app"), "Destination": "/home/plutainer/app"}]}
+
+    def test_final_receipt_follows_data_and_self_removal(self):
+        worker = self.prepare()
+        stages = []
+        def receipt(stage, error=None):
+            stages.append(stage)
+            self.assertFalse(self.root.exists())
+            self.assertFalse(module.AGENT_CONFIG.exists())
+            self.assertFalse(module.AGENT_EXECUTABLE.exists())
+            self.assertFalse(module.AGENT_UNIT.exists())
+            if stage == "cleaned":
+                self.assertTrue(module.UNINSTALL_REQUEST.exists())
+                self.assertTrue(module.UNINSTALL_EXECUTABLE.exists())
+            else:
+                self.assertFalse(self.config_dir.exists())
+                self.assertFalse(module.UNINSTALL_EXECUTABLE.exists())
+                self.assertFalse(module.UNINSTALL_UNIT.exists())
+        worker.receipt = receipt
+        worker.run()
+        self.assertEqual(stages, ["cleaned", "completed"])
+        self.assertEqual((self.base / "unrelated.txt").read_text(), "keep")
+        module.systemctl.assert_any_call("disable", "--now", "electric-cherry-agent.service")
+        module.systemctl.assert_any_call("disable", "electric-cherry-uninstall.service")
+
+    def test_handoff_is_durable_and_never_reports_operation_completed(self):
+        worker = self.prepare()
+        self.assertEqual(worker.job["operationId"], "delete-one")
+        self.assertNotIn("credential", worker.job)
+        self.assertTrue(self.agent.stopping.is_set())
+        module.systemctl.assert_any_call("enable", "--now", "--no-block", "electric-cherry-uninstall.service")
+        self.assertEqual(self.agent.journal.get("delete-one")["state"], "running")
+
+    def test_failed_checkpoint_cannot_leave_a_runnable_uninstall_request(self):
+        with patch.object(self.agent.journal, "checkpoint", side_effect=OSError("journal unavailable")):
+            result = self.agent.execute(self.operation)
+        self.assertEqual(result["state"], "failed")
+        self.assertFalse(module.UNINSTALL_REQUEST.exists())
+        module.systemctl.assert_not_called()
+
+    def test_pending_cleanup_resumes_after_controller_unavailable(self):
+        worker = self.prepare()
+        worker.receipt = lambda *_: (_ for _ in ()).throw(module.AgentError("offline"))
+        with self.assertRaisesRegex(module.AgentError, "offline"):
+            worker.run()
+        self.assertFalse(self.root.exists())
+        self.assertTrue(module.UNINSTALL_REQUEST.exists())
+        self.assertTrue(module.UNINSTALL_EXECUTABLE.exists())
+        self.assertTrue(json.loads(module.UNINSTALL_REQUEST.read_text())["dataRemoved"])
+        before = len(self.docker.calls)
+        resumed = module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
+        stages = []
+        resumed.receipt = lambda stage, error=None: stages.append(stage)
+        resumed.run()
+        self.assertEqual(stages, ["cleaned", "completed"])
+        self.assertEqual(len(self.docker.calls), before)
+
+    def test_lost_final_receipt_retries_after_all_files_are_gone(self):
+        worker = self.prepare()
+        stages = []
+        def receipt(stage, error=None):
+            stages.append(stage)
+            if stage == "completed":
+                self.assertFalse(self.config_dir.exists())
+                if stages.count("completed") == 1:
+                    raise module.AgentError("lost reply")
+        worker.receipt = receipt
+        with patch.object(module.time, "sleep") as wait:
+            worker.run()
+        self.assertEqual(stages, ["cleaned", "completed", "completed"])
+        wait.assert_called_once_with(15)
+
+    def test_partial_self_removal_retries_without_its_unit_or_executable(self):
+        worker = self.prepare()
+        stages = []
+        worker.receipt = lambda stage, error=None: stages.append(stage)
+        original = shutil.rmtree
+        attempts = 0
+        def interrupted(path, *args, **kwargs):
+            nonlocal attempts
+            if Path(path) == self.config_dir:
+                attempts += 1
+                if attempts == 1:
+                    self.assertFalse(module.UNINSTALL_UNIT.exists())
+                    self.assertFalse(module.UNINSTALL_EXECUTABLE.exists())
+                    raise OSError("temporarily busy")
+            return original(path, *args, **kwargs)
+        with patch.object(module.shutil, "rmtree", side_effect=interrupted), patch.object(module.time, "sleep") as wait:
+            worker.run()
+        self.assertEqual(stages, ["cleaned", "failed", "completed"])
+        self.assertFalse(self.config_dir.exists())
+        wait.assert_called_once_with(15)
+
+    def test_partial_tree_removal_recovers_without_deleted_marker(self):
+        worker = self.prepare()
+        original = shutil.rmtree
+        def interrupted(path, *args, **kwargs):
+            if Path(path) == self.root:
+                (self.root / module.ROOT_MARKER).unlink()
+                self.game.unlink()
+                raise OSError("power loss")
+            return original(path, *args, **kwargs)
+        worker.receipt = lambda *_: None
+        with patch.object(module.shutil, "rmtree", side_effect=interrupted):
+            with self.assertRaisesRegex(OSError, "power loss"):
+                worker.run()
+        self.assertTrue(json.loads(module.UNINSTALL_REQUEST.read_text())["removalStarted"])
+        self.assertFalse((self.root / module.ROOT_MARKER).exists())
+        resumed = module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
+        resumed.receipt = lambda *_: None
+        resumed.run()
+        self.assertFalse(self.root.exists())
+
+    def test_replaced_data_root_cannot_be_deleted_on_resume(self):
+        self.prepare()
+        self.root.rename(self.base / "original-data")
+        self.root.mkdir()
+        (self.root / "precious.txt").write_text("keep")
+        with self.assertRaisesRegex(module.AgentError, "changed"):
+            module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
+        self.assertTrue((self.root / "precious.txt").exists())
+
+    def test_only_owned_containers_and_unused_recorded_images_are_removed(self):
+        aliases = {"example/runtime:one": "image-one", "example/runtime:shared": "image-shared", "example/runtime:changed": "image-old"}
+        module.atomic_json(self.root / "agent" / "runtime-images.json", aliases)
+        self.docker.images = {"example/runtime:one": {"Id": "image-one"}, "example/runtime:shared": {"Id": "image-shared"}, "example/runtime:changed": {"Id": "image-new"}, "example/unrelated:one": {"Id": "image-unrelated"}}
+        self.docker.items = {"ours": self.container("ours"), "other": self.container("other", host="host-two", root=self.base / "different-install", image="image-shared")}
+        worker = self.prepare()
+        worker.receipt = lambda *_: None
+        worker.run()
+        self.assertNotIn("ours", self.docker.items)
+        self.assertIn("other", self.docker.items)
+        self.assertNotIn("example/runtime:one", self.docker.images)
+        self.assertIn("example/runtime:shared", self.docker.images)
+        self.assertIn("example/runtime:changed", self.docker.images)
+        self.assertIn("example/unrelated:one", self.docker.images)
+        self.assertFalse(any("prune" in call or "volume" in call for call in self.docker.calls))
+
+    def test_unrelated_container_using_root_blocks_any_removal(self):
+        self.docker.items = {"foreign": self.container("foreign", owned=False)}
+        worker = self.prepare()
+        stages = []
+        worker.receipt = lambda stage, error=None: stages.append((stage, error))
+        with self.assertRaisesRegex(module.AgentError, "unrelated container"):
+            worker.run()
+        self.assertEqual(stages[0][0], "failed")
+        self.assertTrue(self.game.exists())
+        self.assertTrue(module.AGENT_EXECUTABLE.exists())
+        self.assertIn("foreign", self.docker.items)
+
+    def test_running_or_unknown_managed_container_blocks_removal(self):
+        worker = self.prepare()
+        worker.receipt = lambda *_: None
+        for state in ({"Running": True, "Status": "running"}, {"Running": False, "Status": "restarting"}, {}):
+            with self.subTest(state=state):
+                self.docker.items = {"ours": {**self.container("ours"), "State": state}}
+                with self.assertRaisesRegex(module.AgentError, "still running or its state is unknown"):
+                    worker.run()
+                self.assertTrue(self.game.exists())
+                self.assertIn("ours", self.docker.items)
+        self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
+
+    def test_legacy_container_requires_exact_managed_bind_mount(self):
+        worker = self.prepare()
+        legacy = self.container("legacy")
+        legacy["Config"]["Labels"].pop(module.LABEL + ".host")
+        self.assertTrue(worker.owned_container(legacy))
+        legacy["Mounts"][0]["Source"] = str(self.base / "elsewhere")
+        self.assertFalse(worker.owned_container(legacy))
+
+    def test_missing_marker_and_wrong_host_never_schedule_helper(self):
+        (self.root / module.ROOT_MARKER).unlink()
+        self.assertFalse(self.agent.can_uninstall())
+        result = self.agent.execute(self.operation)
+        self.assertEqual(result["state"], "failed")
+        module.systemctl.assert_not_called()
+        self.assertTrue(self.game.exists())
+
+    def test_receipt_path_and_identity_cannot_be_redirected(self):
+        for key, value in (("receiptUrl", "https://attacker.example"), ("hostId", "other"), ("operationId", "other")):
+            with self.subTest(key=key):
+                operation = {**self.operation, "id": "check-" + key, "payload": {**self.operation["payload"], "operationId": "check-" + key, "receiptUrl": "/api/agent/decommission/check-" + key, key: value}}
+                self.assertEqual(self.agent.execute(operation)["state"], "failed")
+        module.systemctl.assert_not_called()
+
+    def test_symlink_inside_owned_tree_is_unlinked_without_following(self):
+        outside = self.base / "unrelated.txt"
+        (self.root / "libraries" / "symlink").symlink_to(outside)
+        worker = self.prepare()
+        worker.receipt = lambda *_: None
+        worker.run()
+        self.assertEqual(outside.read_text(), "keep")
+
+    def test_unsafe_roots_and_installation_overlap_are_rejected(self):
+        for value in ("/", "/var/lib", "/srv", "/var/lib/../lib", "/var//lib/anything", str(self.config_dir), str(self.bin_dir)):
+            with self.subTest(value=value):
+                with self.assertRaises(module.AgentError):
+                    module.uninstall_root(value)
+
+    def test_root_mount_or_nested_mount_cannot_be_recursively_deleted(self):
+        original = Path.read_text
+        for mount in (self.root, self.root / "libraries"):
+            def read_text(path, *args, **kwargs):
+                if str(path) == "/proc/1/mountinfo":
+                    return "21 1 8:1 / " + str(mount) + " rw - ext4 /dev/sda1 rw\n"
+                return original(path, *args, **kwargs)
+            with patch.object(Path, "read_text", read_text):
+                with self.assertRaisesRegex(module.AgentError, "Unmount"):
+                    module.uninstall_root(str(self.root))
+
+    def test_marker_directory_is_rejected_before_reading(self):
+        marker = self.root / module.ROOT_MARKER
+        marker.unlink()
+        marker.mkdir()
+        with self.assertRaisesRegex(module.AgentError, "regular file"):
+            module.uninstall_ownership(self.config)
+
+    def test_boot_reconciles_handoff_instead_of_marking_complete(self):
+        self.prepare()
+        self.agent.journal.close()
+        self.agent = module.Agent(self.config)
+        self.assertEqual(self.agent.journal.get("delete-one")["state"], "unknown")
+        result = self.agent.execute(self.operation)
+        self.assertEqual(result["state"], "running")
+        self.assertEqual(json.loads(module.UNINSTALL_REQUEST.read_text())["operationId"], "delete-one")
+
+
+class OwnershipPathTests(unittest.TestCase):
+    def test_root_ownership_group_writes_and_symlinks_are_rejected(self):
+        for mode, uid in ((stat.S_IFDIR | 0o777, 0), (stat.S_IFDIR | 0o755, 1000), (stat.S_IFLNK | 0o777, 0)):
+            with self.subTest(mode=mode, uid=uid):
+                value = os.stat_result((mode, 1, 1, 1, uid, 0, 0, 0, 0, 0))
+                with patch.object(Path, "lstat", return_value=value):
+                    with self.assertRaisesRegex(module.AgentError, "only by root"):
+                        module.protected_path("/var/lib/electric-cherry")
+
+
+if __name__ == "__main__":
+    unittest.main()

docs/authentication.md

@@ -85,6 +85,9 @@ endpoint or accept passwords as command arguments.
 Host agents retain enrollment/host-token authentication under `/api/agent/`.
 Installer and release files remain public so a new machine can enroll. A browser
 login never substitutes for a host credential, and host tokens cannot manage EC.
+Host deletion uses a separate operation-scoped receipt token so the uninstall
+worker can confirm removal and retry its acknowledgment after host credentials
+are revoked. That token authorizes only the corresponding cleanup receipt.
 
 Game reporters must send a separate secret of at least 32 characters configured
 as `EC_TELEMETRY_TOKEN`:

docs/native-hosts.md

@@ -35,7 +35,10 @@ A fresh destination host downloads centrally stored game files without manual
 copying or per-container setup. EC does not supply the game content itself.
 
 The default host root is `/var/lib/electric-cherry`; installer `--data-dir` changes
-it. Data is separated into `libraries/<game>/<version>/content`,
+it. Use a dedicated, root-owned directory. For a separate disk, use a child of its
+mount point (for example `/mnt/games/electric-cherry`), with no nested mounts, so
+host deletion can remove the entire directory. The installer records ownership
+and refuses to adopt unrelated files. Data is separated into `libraries/<game>/<version>/content`,
 `instances/<UUID>/<generation>/app`, and `agent/operations.sqlite3`. Base libraries
 are mounted read-only and instance deletion removes only that instance's files.
 The controller stores large files under `GAME_FILES_ROOT` (default beside the
@@ -74,12 +77,15 @@ explicit installer `--reenroll` when local credentials already exist.
 API outline:
 
 - `GET /api/hosts/:id`: host details, operation history and deletion blockers.
-- `DELETE /api/hosts/:id`: administrator removes an unused host and revokes access;
-  returns `409` if servers, reservations or unresolved delivered operations remain.
+- `DELETE /api/hosts/:id`: administrator requests cleanup of an unused host;
+  returns `202` while remote removal is pending, `200` for an unenrolled host,
+  or `409` when the agent needs updating or unresolved ownership remains.
 - `POST /api/agent/enroll`: consume enrollment token and return host credentials.
 - `POST /api/agent/heartbeat`: authenticated inventory, current operation and caches.
 - `GET /api/agent/operations`: deliver/reconcile durable operation IDs.
 - `POST /api/agent/operations/:id`: authenticated acceptance, progress or result.
+- `POST /api/agent/decommission/:id`: cleanup worker reports file removal and
+  agent uninstall using its own operation-scoped receipt credential.
 - `POST /api/game-files`: administrator creates a folder upload with game, name,
   version, optional description and `files: [{path, size, lastModified}]`.
 - `GET /api/game-files/:id/upload`: durable per-file offsets and publication status.
@@ -133,13 +139,34 @@ Administrators can choose **Delete host** on a host card or at the top of its
 detail page. The confirmation lists anything that must be resolved first:
 registered servers, reserved ports, or operations already sent to the agent.
 Drain the host or remove its servers, then let outstanding operations reconcile.
-An unused host can be deleted before enrollment or while offline.
+An unenrolled host is removed immediately. An enrolled host stays visible as
+**Deleting** until its agent confirms cleanup. Offline hosts wait for their next
+connection; their absence is never treated as proof of removal. Existing agents
+need the 1.1.0 installer: generate a fresh command under **Agent access**, rerun it
+on the host, and wait for a heartbeat advertising uninstall support.
 
-Deleting a host removes it from the controller's host list and deployment
-choices, revokes its credentials, and invalidates unused installer links.
-Queued commands that have never been sent to the agent are cancelled. Operation
-history and published libraries are retained; the agent and game files remain on
-the machine. Adding the machine again requires a new host enrollment.
+Deletion blocks new operations, cancels queued commands that were never delivered,
+and invalidates unused installer links. A separate cleanup service removes the
+host's downloaded game libraries, instance files, agent journal and configuration,
+owned containers, and recorded runtime images unused by other containers. It
+stops and removes the agent service and executable, then removes its own files.
+The controller hides the host and revokes its ordinary credentials only after
+receiving the final uninstall receipt. Adding the machine again requires a new
+host enrollment.
+
+Game folders uploaded through Electric Cherry belong to the controller. Those
+original uploads remain available for other hosts and future deployments; only
+the deleted host's copies are removed. Controller operation history is also kept.
+Unrelated containers, files and shared dependencies such as Docker and Python
+are preserved. Historical entries in shared system logs are not erased.
+
+Cleanup retries after connection loss or partial file removal. Failures before
+handoff offer **Retry cleanup** after the installation is repaired; failures in
+the independent worker retry automatically. The worker stays installed until
+the controller acknowledges data cleanup. After removing its final files it
+retries the completion receipt from memory. A power loss in that final window
+can leave the controller awaiting confirmation even though removal succeeded;
+inspect the machine if that status persists. The controller does not guess.
 
 ## Libraries and ports
 
@@ -198,7 +225,9 @@ Automated tests cover enrollment/authentication, rotation/revocation, host
 readiness, scoped ports, immutable library publication, streaming/ranges,
 checksums, interrupted downloads, instance generation ownership, local occupancy
 holds, controller restart/reconnect, ambiguous results, occupied campaign staging,
-activation before images, collections and draining. Docker/UDP and host
+activation before images, collections and draining. Host deletion tests cover
+ownership guards, interrupted cleanup, uninstall receipts, credential revocation
+and retention of controller uploads/history. Docker/systemd/UDP and host
 observations are simulated in these tests.
 
 A real Debian/Docker/Plutonium acceptance run is still required before production
@@ -213,6 +242,10 @@ coverage is not evidence that a real T5/T6 server has been deployed here. Run:
   agent activation, and during file activation before restart; verify that the
   agent holds disruption.
 - Drain occupied lobbies and verify shared libraries survive their retirement.
+- Delete a drained host; verify its game copies, data directory, agent and cleanup
+  services/configuration/executables are removed, controller uploads/history stay,
+  and unrelated Docker workloads continue. Repeat with connection loss during
+  cleanup and after the final receipt, and a reboot during data removal.
 - Corrupt/interrupted library downloads, resumed transfer and verified promotion.
 - Controller restart, agent restart and network loss before execution, during
   execution and after execution but before acknowledgement. Verify the same

src/lib/hosts.ts

@@ -1,13 +1,27 @@
 export type HostMode = 'active' | 'draining' | 'disabled';
-export type HostReadiness = 'preparing' | 'needs-runtime' | 'needs-game-files' | 'ready' | 'degraded' | 'draining' | 'disabled';
+export type HostReadiness = 'preparing' | 'needs-runtime' | 'needs-game-files' | 'ready' | 'degraded' | 'draining' | 'disabled' | 'deleting';
 export type HostOperationState = 'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown';
 
 export function hostReadinessLabel(readiness: HostReadiness): string {
     return { preparing: 'Preparing', 'needs-runtime': 'Needs runtime', 'needs-game-files': 'Needs game files',
-        ready: 'Ready', degraded: 'Degraded', draining: 'Draining', disabled: 'Disabled' }[readiness];
+        ready: 'Ready', degraded: 'Degraded', draining: 'Draining', disabled: 'Disabled', deleting: 'Deleting' }[readiness];
 }
 
 export interface HostDeletion { allowed: boolean; blockers: string[] }
+export interface HostDecommission {
+    operationId: string; stage: 'queued' | 'running' | 'cleaned' | 'failed' | 'completed';
+    requestedAt: string; updatedAt: string; error: string | null; retryable: boolean;
+}
+export function hostDeletionMessage(host: Pick<Host, 'connected' | 'decommission'>): string {
+    if (!host.decommission) return '';
+    switch (host.decommission.stage) {
+        case 'queued': return host.connected ? 'Host cleanup is queued.' : 'Waiting for the host to reconnect before removing its files and agent.';
+        case 'running': return 'Removing host files, cached game libraries, and the agent installation.';
+        case 'cleaned': return 'Host data is removed. Waiting for confirmation that the agent has uninstalled itself.';
+        case 'failed': return 'Cleanup needs attention. Complete removal has not been confirmed.';
+        case 'completed': return 'Host files and agent removal confirmed.';
+    }
+}
 
 export interface HostLibrary { id: string; state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; progress?: number; error?: string }
 export interface HostInventory {
@@ -16,6 +30,7 @@ export interface HostInventory {
     storageTotal?: number; storageAvailable?: number; dockerVersion?: string; dockerRunning?: boolean;
     directoriesWritable?: boolean; runtimeImages?: string[]; libraries?: HostLibrary[];
     currentOperation?: string | null; [key: string]: unknown;
+    capabilities?: string[];
 }
 export interface Host {
     id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number;
@@ -23,6 +38,7 @@ export interface Host {
     lastHeartbeat: string | null; connected: boolean; connectionState: 'online' | 'offline';
     readiness: HostReadiness; readinessReasons: string[]; inventory: HostInventory;
     enrolled: boolean; credentialActive: boolean;
+    decommission: HostDecommission | null;
 }
 export interface HostOperation {
     id: string; hostId: string; kind: string; instanceId: string | null; generation: number | null;

src/lib/server/hosts/index.ts

@@ -1,8 +1,8 @@
-import { createHash, randomBytes, randomUUID } from 'node:crypto';
+import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
 import { isIP } from 'node:net';
 import { db } from '../db/index.js';
 import { ensureHostSchema } from './schema.js';
-import type { Host, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
+import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
 
 export const AGENT_PROTOCOL_VERSION = 1;
 export const HEARTBEAT_MAX_AGE_MS = 90_000;
@@ -11,6 +11,14 @@ const digest = (token: string) => createHash('sha256').update(token).digest('hex
 function database() { const handle = db(); ensureHostSchema(handle); return handle; }
 type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number };
 const hostSql = `SELECT h.*, EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id = h.id AND c.revoked_at IS NULL) AS credential_active FROM ec_hosts h`;
+type DecommissionRow = { host_id: string; operation_id: string; token_hash: string; stage: HostDecommission['stage']; requested_at: string; updated_at: string; cleaned_at: string | null; error: string | null };
+function decommissionRow(hostId: string): DecommissionRow | undefined {
+    return database().prepare('SELECT * FROM ec_host_decommissions WHERE host_id=?').get(hostId) as DecommissionRow | undefined;
+}
+function presentDecommission(row: DecommissionRow | undefined): HostDecommission | null {
+    return row ? { operationId: row.operation_id, stage: row.stage, requestedAt: row.requested_at, updatedAt: row.updated_at,
+        error: row.error, retryable: row.stage === 'failed' && getHostOperation(row.operation_id)?.state === 'failed' } : null;
+}
 function hostFromRow(row: HostRow): Host {
     const inventory: HostInventory = JSON.parse(row.inventory);
     const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
@@ -29,11 +37,13 @@ function hostFromRow(row: HostRow): Host {
     if (!libraryReady) reasons.push('Download and verify a game-file library.');
     if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
     if (row.mode === 'disabled') reasons.push('Host is disabled.');
+    const decommission = presentDecommission(decommissionRow(row.id));
+    if (decommission) reasons.push('Host deletion is in progress; waiting for confirmed cleanup and agent removal.');
     return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max,
         bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes,
         createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline',
-        readiness: row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
-        readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active };
+        readiness: decommission ? 'deleting' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
+        readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active, decommission };
 }
 export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
 export function getHost(id: string): Host | null {
@@ -43,14 +53,18 @@ export function getHost(id: string): Host | null {
 function requireHost(id: string): Host { const host = getHost(id); if (!host) throw new Error('Host was not found.'); return host; }
 
 export class HostDeletionError extends Error {
-    constructor(message: string, readonly status: 404 | 409, readonly blockers: string[] = []) { super(message); }
+    constructor(message: string, readonly status: 400 | 401 | 404 | 409, readonly blockers: string[] = []) { super(message); }
 }
 
 /** Controller records can be removed only after all remote instance ownership is resolved. */
 export function hostDeletionStatus(id: string): HostDeletion {
-    if (!getHost(id)) throw new HostDeletionError('Host not found.', 404);
+    const host = getHost(id);
+    if (!host) throw new HostDeletionError('Host not found.', 404);
+    if (host.decommission && !host.decommission.retryable) return { allowed: true, blockers: [] };
     const handle = database();
     const blockers: string[] = [];
+    if (host.enrolled && !host.inventory.capabilities?.includes('host.decommission')) blockers.push('Update this host agent before deleting it: generate an installer from Agent access, rerun it on this host, and wait for a fresh heartbeat.');
+    if (host.enrolled && !host.credentialActive) blockers.push('Re-enroll this host so its agent can receive and complete the cleanup request.');
     const instances = handle.prepare('SELECT server FROM native_instances WHERE host_id=? AND retired_at IS NULL ORDER BY server').all(id) as { server: string }[];
     if (instances.length) {
         const names = instances.slice(0, 5).map(instance => instance.server).join(', ') + (instances.length > 5 ? ', …' : '');
@@ -70,20 +84,68 @@ export function hostDeletionStatus(id: string): HostDeletion {
     return { allowed: blockers.length === 0, blockers };
 }
 
-/** Retain references and audit history; deletion never sends remote cleanup commands. */
-export function deleteHost(id: string): void {
+/** Keep the host visible and authenticated until its independent cleanup worker proves removal. */
+export function deleteHost(id: string): { deleted: boolean; operationId?: string } {
     const handle = database();
-    handle.transaction(() => {
+    return handle.transaction(() => {
+        const host = getHost(id);
+        if (!host) throw new HostDeletionError('Host not found.', 404);
+        if (host.decommission && !host.decommission.retryable) return { deleted: false, operationId: host.decommission.operationId };
         const status = hostDeletionStatus(id);
         if (!status.allowed) throw new HostDeletionError(status.blockers.join(' '), 409, status.blockers);
         const timestamp = now();
-        const reason = 'Host deleted before this operation was delivered to the agent.';
+        const reason = 'Host deletion requested before this operation was delivered to the agent.';
         handle.prepare(`INSERT INTO ec_host_operation_events(operation_id,state,created_at,error)
             SELECT id,'failed',?,? FROM ec_host_operations WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL`).run(timestamp, reason, id);
         handle.prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL").run(reason, timestamp, id);
-        handle.prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(timestamp, id);
         handle.prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, id);
+        if (!host.enrolled) {
             handle.prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL").run(timestamp, id);
+            return { deleted: true };
+        }
+        const operationId = randomUUID();
+        const token = 'ec_cleanup_' + randomBytes(32).toString('base64url');
+        const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` };
+        handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at) VALUES(?,?,'host.decommission',?,'pending',?,?)").run(operationId, id, JSON.stringify(payload), timestamp, timestamp);
+        handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp);
+        handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?)
+            ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
+            .run(id, operationId, digest(token), timestamp, timestamp);
+        handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id);
+        return { deleted: false, operationId };
+    }).immediate();
+}
+
+export function receiveDecommissionReceipt(operationId: string, token: string, input: Record<string, unknown>): { acknowledged: true } {
+    if (!token || token.length > 200) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
+    return database().transaction(() => {
+        const row = database().prepare('SELECT * FROM ec_host_decommissions WHERE operation_id=?').get(operationId) as DecommissionRow | undefined;
+        if (!row || !timingSafeEqual(Buffer.from(row.token_hash, 'hex'), Buffer.from(digest(token), 'hex'))) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
+        if (!['cleaned', 'completed', 'failed'].includes(input.stage as string)) throw new HostDeletionError('Invalid cleanup receipt stage.', 400);
+        if (row.stage === 'completed') return { acknowledged: true as const };
+        const operation = getHostOperation(operationId)!;
+        if (!['accepted','running','unknown'].includes(operation.state)) throw new HostDeletionError('Cleanup has not been accepted by the host.', 409);
+        const timestamp = now();
+        if (input.stage === 'failed') {
+            database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE host_id=?")
+                .run(typeof input.error === 'string' ? input.error.slice(0, 2000) : 'Host cleanup needs attention. The cleanup worker will retry.', timestamp, row.host_id);
+            return { acknowledged: true as const };
+        }
+        if (input.stage === 'cleaned') {
+            database().prepare("UPDATE ec_host_decommissions SET stage='cleaned',cleaned_at=COALESCE(cleaned_at,?),error=NULL,updated_at=? WHERE host_id=?").run(timestamp, timestamp, row.host_id);
+            return { acknowledged: true as const };
+        }
+        if (!row.cleaned_at) throw new HostDeletionError('The host must confirm file cleanup before confirming agent removal.', 409);
+        database().prepare("UPDATE ec_host_decommissions SET stage='completed',error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id);
+        database().prepare("UPDATE ec_host_operations SET state='completed',result=?,error=NULL,updated_at=? WHERE id=?").run(JSON.stringify({ filesRemoved: true, agentRemoved: true }), timestamp, operationId);
+        database().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'completed',?)").run(operationId, timestamp);
+        database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(timestamp, row.host_id);
+        database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, row.host_id);
+        if (database().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='host_library_caches'").get()) {
+            database().prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id);
+        }
+        database().prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=?").run(timestamp, row.host_id);
+        return { acknowledged: true as const };
     }).immediate();
 }
 function textField(value: unknown, label: string, max = 200): string {
@@ -102,6 +164,7 @@ export function createHost(input: Record<string, unknown>): Host {
 }
 export function updateHost(id: string, input: Record<string, unknown>): Host {
     const host = requireHost(id);
+    if (host.decommission) throw new Error('Host cleanup is in progress; its settings cannot change.');
     const name = input.name === undefined ? host.name : textField(input.name, 'Host name');
     const mode = input.mode ?? host.mode;
     if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.');
@@ -117,7 +180,8 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
 }
 
 export function createEnrollment(hostId: string, ttlMs = 15 * 60_000): { id: string; hostId: string; token: string; expiresAt: string } {
-    requireHost(hostId);
+    const host = requireHost(hostId);
+    if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup is in progress; enrollment is unavailable.');
     if (!Number.isInteger(ttlMs) || ttlMs <= 0 || ttlMs > 60 * 60_000) throw new Error('Enrollment lifetime must be at most one hour.');
     const token = 'ec_enroll_' + randomBytes(32).toString('base64url'); const id = randomUUID();
     const expiresAt = new Date(Date.now() + ttlMs).toISOString();
@@ -143,6 +207,7 @@ export function validateInventory(value: unknown): HostInventory {
     }
     for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
     if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
+    if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
     if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library =>
         !library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.');
     return inventory;
@@ -169,7 +234,8 @@ export function authenticateCredential(token: string): Host | null {
     return getHost(row.host_id);
 }
 export function revokeCredentials(hostId: string): void {
-    requireHost(hostId);
+    const host = requireHost(hostId);
+    if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup needs its credentials until removal is confirmed.');
     database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(now(), hostId);
 }
 export function rotateCredential(hostId: string, nextCredential?: string): string {
@@ -207,6 +273,8 @@ export function getHostOperation(id: string): HostOperation | null { const row =
 export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
 export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation {
     const host = requireHost(hostId);
+    if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.');
+    if (kind === 'host.decommission') throw new Error('Use the host deletion workflow to request cleanup.');
     if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
     if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
     if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.');
@@ -245,6 +313,7 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
         requireHost(hostId);
         const operation = getHostOperation(id);
         if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.');
+        if (operation.kind === 'host.decommission' && input.state === 'completed') throw new Error('Host removal requires the cleanup worker completion receipt.');
         if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.');
         if (['completed','failed'].includes(operation.state)) return operation;
         const state = input.state as HostOperationState;
@@ -253,6 +322,10 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
         const result = input.result === undefined ? null : JSON.stringify(input.result);
         if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.');
         database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=? WHERE id=?').run(state, result, error, now(), id);
+        if (operation.kind === 'host.decommission') {
+            if (state === 'failed') database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?").run(error ?? 'The agent could not start cleanup. Repair the installation, then retry deletion.', now(), id);
+            else database().prepare("UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')").run(now(), id);
+        }
         if (operation.state !== state) database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)').run(id, state, now(), error);
         return getHostOperation(id)!;
     })();
@@ -270,6 +343,7 @@ export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise
 
 export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean {
     const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress;
+    if (host.decommission) return false;
     if (candidate < host.portMin || candidate > host.portMax) return false;
     const rows = database().prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND protocol=? AND port=?').all(hostId, protocol, candidate) as { instance_id: string; bind_address: string }[];
     return !rows.some(row => row.instance_id !== instanceId && (row.bind_address === bind || ['0.0.0.0','::'].includes(bind) || ['0.0.0.0','::'].includes(row.bind_address)));
@@ -281,6 +355,7 @@ export function nextHostPort(hostId: string, additionalPorts: number[] = [], bin
 }
 export function reserveHostPort(hostId: string, instanceId: string, options: { port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): number {
     const host = requireHost(hostId); const bind = options.bindAddress ?? host.bindAddress; const protocol = options.protocol ?? 'udp';
+    if (host.decommission) throw new Error('Host cleanup is in progress; new port reservations are blocked.');
     if (!isIP(bind)) throw new Error('Bind address must be an IP address.');
     if (!['udp','tcp'].includes(protocol)) throw new Error('Port protocol must be UDP or TCP.');
     if (!instanceId) throw new Error('An immutable instance ID is required.');

src/lib/server/hosts/schema.ts

@@ -40,6 +40,13 @@ export function ensureHostSchema(handle: Database.Database): void {
             PRIMARY KEY(host_id,bind_address,protocol,port)
         );
         CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id);
+        CREATE TABLE IF NOT EXISTS ec_host_decommissions (
+            host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id),
+            operation_id TEXT NOT NULL UNIQUE REFERENCES ec_host_operations(id),
+            token_hash TEXT NOT NULL, stage TEXT NOT NULL DEFAULT 'queued'
+                CHECK(stage IN ('queued','running','cleaned','failed','completed')),
+            requested_at TEXT NOT NULL, updated_at TEXT NOT NULL, cleaned_at TEXT, error TEXT
+        );
     `);
     const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[];
     if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');

src/lib/ui/components/DeleteHostDialog.svelte

@@ -4,7 +4,7 @@
     import Dialog from './primitives/Dialog.svelte';
 
     let { open = $bindable(false), host, ondeleted }:
-        { open?: boolean; host: Pick<Host, 'id' | 'name'> | null; ondeleted: () => void | Promise<void> } = $props();
+        { open?: boolean; host: Pick<Host, 'id' | 'name'> | null; ondeleted: (result: { deleted: boolean; operationId?: string }) => void | Promise<void> } = $props();
     let deletion = $state<HostDeletion | null>(null);
     let checking = $state(false); let deleting = $state(false); let error = $state('');
 
@@ -36,19 +36,20 @@
                 throw new Error(body.error ?? body.message ?? 'Could not delete this host.');
             }
             open = false;
-            await ondeleted();
+            await ondeleted(body);
         } catch (cause) { error = (cause as Error).message; }
         finally { deleting = false; }
     }
 </script>
 
 <Dialog bind:open title={`Delete ${host?.name ?? 'host'}?`} busy={deleting}
-    description="Remove this host from Electric Cherry and revoke its agent access and unused installer links.">
+    description="Permanently remove Electric Cherry's files from this host, including its downloaded game files, then uninstall the agent.">
     <div class="content">
-        <p>The agent and game files stay on the machine. Published game libraries and operation history are kept. Commands that have never been sent to the host will be cancelled.</p>
+        <p>This deletes the host's game-library copies, server files, agent data, credentials, and service installation. Electric Cherry's uploaded libraries on the controller and its operation history are kept.</p>
+        <p>The host remains visible until cleanup is confirmed. Offline hosts wait for reconnection. Unsent commands will be cancelled.</p>
         {#if checking}<p role="status">Checking servers and unfinished operations…</p>
         {:else if deletion && !deletion.allowed}<div role="status" class="blocked"><strong>This host still has work to finish:</strong><ul>{#each deletion.blockers as blocker}<li>{blocker}</li>{/each}</ul><p>Open <a href={`/hosts/${host?.id}`} onclick={() => open = false}>host details</a> to review its operations or drain its servers.</p></div>
-        {:else if deletion?.allowed}<p>This host is ready to delete. Adding it again requires a new enrollment.</p>{/if}
+        {:else if deletion?.allowed}<p>Cleanup cannot be undone once it starts. Adding this machine again requires installing and enrolling the agent again.</p>{/if}
         {#if error}<p role="alert" class="ui-error">{error}</p>{/if}
     </div>
     {#snippet actions()}

src/routes/api/agent/decommission/[id]/+server.ts

@@ -0,0 +1,14 @@
+import { json } from '@sveltejs/kit';
+import { HostDeletionError, receiveDecommissionReceipt } from '$lib/server/hosts/index.js';
+import { objectBody, requireAgentTransport } from '$lib/server/hosts/http.js';
+
+/** The uninstall worker retains only this narrowly scoped credential after removing the agent. */
+export async function POST({ request, params }: { request: Request; params: { id: string } }) {
+    requireAgentTransport(request);
+    const token = request.headers.get('authorization')?.match(/^Bearer ([A-Za-z0-9_-]+)$/i)?.[1] ?? '';
+    try { return json(receiveDecommissionReceipt(params.id, token, await objectBody(request, 8192)), { headers: { 'Cache-Control': 'no-store' } }); }
+    catch (cause) {
+        if (cause instanceof HostDeletionError) return json({ error: cause.message }, { status: cause.status });
+        throw cause;
+    }
+}

src/routes/api/hosts/[id]/+server.ts

@@ -15,7 +15,7 @@ export async function PATCH({ request, params }: Event) {
 
 export function DELETE({ request, params }: Event) {
     requireManagementOrigin(request);
-    try { deleteHost(params.id); return json({ deleted: true }); }
+    try { const result = deleteHost(params.id); return json(result, { status: result.deleted ? 200 : 202 }); }
     catch (cause) {
         if (cause instanceof HostDeletionError) return json({ error: cause.message, blockers: cause.blockers }, { status: cause.status });
         return hostApiError(cause);

src/routes/hosts/+page.svelte

@@ -2,7 +2,7 @@
     import { onMount } from 'svelte';
     import { page } from '$app/stores';
     import type { Host } from '$lib/hosts.js';
-    import { hostReadinessLabel } from '$lib/hosts.js';
+    import { hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Button from '$ui/components/primitives/Button.svelte';
     import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
@@ -18,8 +18,8 @@
         try { const response = await fetch('/api/hosts'); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); if (sequence === loadSequence) { hosts = body.hosts; error = ''; } }
         catch (cause) { if (sequence === loadSequence) error = (cause as Error).message; } finally { if (sequence === loadSequence) loading = false; }
     }
-    async function deleted() {
-        notice = `${deletingHost?.name ?? 'Host'} deleted.`;
+    async function deleted(result: { deleted: boolean }) {
+        notice = result.deleted ? `${deletingHost?.name ?? 'Host'} deleted.` : `Cleanup requested for ${deletingHost?.name ?? 'host'}. It remains listed until its files and agent are removed.`;
         if (enrollment?.hostId === deletingHost?.id) enrollment = null;
         deletingHost = null;
         await load();
@@ -51,7 +51,7 @@
         {#if enrollment}<div class="enrollment" role="status"><h3>Install the agent</h3><p>This single-use command expires {new Date(enrollment.expiresAt).toLocaleString()}. Run it with administrator access on your game host.</p><pre>{enrollment.command}</pre><a href={`/hosts/${enrollment.hostId}`}>Open host setup →</a></div>{/if}
     </section>{:else}<p>Administrators manage host setup, runtime preparation, and credentials.</p>{/if}
     {#if loading}<p role="status">Loading hosts…</p>{:else if !hosts.length}<p>No hosts connected yet.{canManage ? ' Add a host to begin.' : ''}</p>{:else}
-        <div class="host-grid">{#each hosts as host (host.id)}<article class="panel host"><div class="row"><h2><a href={`/hosts/${host.id}`}>{host.name}</a></h2><span class:ready={host.readiness === 'ready'} class="state">{hostReadinessLabel(host.readiness)}</span></div><p>{host.hostname || 'Awaiting enrollment'} · {host.connectionState}</p><p>{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}</p><p>Game ports {host.portMin}–{host.portMax}</p>{#if host.readinessReasons.length}<p class="reason">{host.readinessReasons[0]}</p>{/if}<div class="host-actions"><Button size="sm" href={`/hosts/${host.id}`}>Manage host</Button>{#if host.readiness === 'needs-game-files'}<Button size="sm" href="/game-files">Choose game files</Button>{/if}{#if canManage}<Button size="sm" variant="danger" onclick={() => { deletingHost = host; deleteOpen = true; }}>Delete host</Button>{/if}</div></article>{/each}</div>
+        <div class="host-grid">{#each hosts as host (host.id)}<article class="panel host"><div class="row"><h2><a href={`/hosts/${host.id}`}>{host.name}</a></h2><span class:ready={host.readiness === 'ready'} class="state">{hostReadinessLabel(host.readiness)}</span></div><p>{host.hostname || 'Awaiting enrollment'} · {host.connectionState}</p><p>{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}</p><p>Game ports {host.portMin}–{host.portMax}</p>{#if host.decommission}<p class="reason">{hostDeletionMessage(host)}</p>{#if host.decommission.error}<p class="ui-error">{host.decommission.error}</p>{/if}{:else if host.readinessReasons.length}<p class="reason">{host.readinessReasons[0]}</p>{/if}<div class="host-actions"><Button size="sm" href={`/hosts/${host.id}`}>{host.decommission ? 'View cleanup' : 'Manage host'}</Button>{#if host.readiness === 'needs-game-files'}<Button size="sm" href="/game-files">Choose game files</Button>{/if}{#if canManage && (!host.decommission || host.decommission.retryable)}<Button size="sm" variant="danger" onclick={() => { deletingHost = host; deleteOpen = true; }}>{host.decommission ? 'Retry cleanup' : 'Delete host'}</Button>{/if}</div></article>{/each}</div>
     {/if}
 </section>
 <DeleteHostDialog bind:open={deleteOpen} host={deletingHost} ondeleted={deleted} />

src/routes/hosts/[id]/+page.svelte

@@ -3,7 +3,7 @@
     import { page } from '$app/stores';
     import { goto } from '$app/navigation';
     import type { Host, HostOperation, HostMode } from '$lib/hosts.js';
-    import { hostReadinessLabel } from '$lib/hosts.js';
+    import { hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Button from '$ui/components/primitives/Button.svelte';
     import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
@@ -14,10 +14,11 @@
     let runtimeImage = $state('ghcr.io/ayymoss/plutainer:latest'); let enrollment = $state<{ command: string; expiresAt: string } | null>(null);
     const endpoint = $derived(`/api/hosts/${$page.params.id}`);
     const canManage = $derived($page.data.user?.role === 'admin');
+    const cleaning = $derived(!!host?.decommission);
     function bytes(value: unknown) { return typeof value === 'number' ? `${(value / 1073741824).toFixed(1)} GiB` : 'Unknown'; }
     async function load(initialize = false) {
         try {
-            const response = await fetch(endpoint); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations;
+            const response = await fetch(endpoint); const body = await response.json(); if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations;
             if (initialize && host) { name = host.name; portMin = host.portMin; portMax = host.portMax; bindAddress = host.bindAddress; publicAddress = host.publicAddress; minimumFreeGB = host.minimumFreeBytes / 1073741824; }
         } catch (cause) { error = (cause as Error).message; } finally { loading = false; }
     }
@@ -33,30 +34,31 @@
 
 <svelte:head><title>{host?.name ?? 'Host'} · Electric Cherry</title></svelte:head>
 <section class="page"><a href="/hosts">← All hosts</a><SectionHeader title={host?.name ?? 'Host'} subtitle="Readiness, inventory, credentials, and operation history.">
-    {#snippet actions()}{#if canManage && host}<Button variant="danger" disabled={busy} onclick={() => deleteOpen = true}>Delete host</Button>{/if}{/snippet}
+    {#snippet actions()}{#if canManage && host && (!cleaning || host.decommission?.retryable)}<Button variant="danger" disabled={busy} onclick={() => deleteOpen = true}>{cleaning ? 'Retry cleanup' : 'Delete host'}</Button>{/if}{/snippet}
 </SectionHeader>
     {#if error}<p class="ui-error" role="alert">{error}</p>{/if}{#if notice}<p role="status">{notice}</p>{/if}
     {#if loading}<p role="status">Loading host…</p>{:else if host}
+        {#if host.decommission}<section class="panel" aria-live="polite"><h2>Host cleanup</h2><p>{hostDeletionMessage(host)}</p>{#if host.decommission.error}<p class="ui-error">{host.decommission.error}</p>{/if}<p>Requested {new Date(host.decommission.requestedAt).toLocaleString()}. This page closes only after the cleanup worker confirms removal.</p>{#if host.decommission.stage === 'cleaned'}<p>The agent is expected to go offline during uninstall. If final confirmation never arrives, inspect the machine; loss of contact alone does not prove completion.</p>{/if}</section>{/if}
         <section class="panel"><div class="row"><h2>{host.connectionState} · {hostReadinessLabel(host.readiness)}</h2><span>Last heartbeat: {host.lastHeartbeat ? new Date(host.lastHeartbeat).toLocaleString() : 'Never'}</span></div>
             {#if host.readinessReasons.length}<ul>{#each host.readinessReasons as reason}<li>{reason}</li>{/each}</ul>{:else}<p>Connected and ready. Each deployment also checks its game library, runtime, and ports.</p>{/if}
             {#if host.readiness === 'needs-game-files'}<p><a href="/game-files">Choose a published game-file version and download it to this host →</a></p>{/if}
-            {#if canManage}<div class="actions"><Button disabled={busy || host.mode === 'active'} onclick={() => mode('active')}>Enable</Button><Button disabled={busy || host.mode === 'draining'} onclick={() => mode('draining')}>Drain host</Button><Button disabled={busy || host.mode === 'disabled'} onclick={() => mode('disabled')}>Disable host</Button></div>{/if}
+            {#if canManage && !cleaning}<div class="actions"><Button disabled={busy || host.mode === 'active'} onclick={() => mode('active')}>Enable</Button><Button disabled={busy || host.mode === 'draining'} onclick={() => mode('draining')}>Drain host</Button><Button disabled={busy || host.mode === 'disabled'} onclick={() => mode('disabled')}>Disable host</Button></div>{/if}
             <p>Draining prevents new deployments and retires instances as they become safe to remove. Disabled hosts receive no new commands; running game containers continue.</p>
         </section>
         <div class="grid"><section class="panel"><h2>Inventory</h2><dl>
             <dt>Hostname</dt><dd>{host.hostname || 'Pending'}</dd><dt>Agent</dt><dd>{host.inventory.agentVersion ?? 'Unknown'}</dd><dt>Platform</dt><dd>{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}</dd><dt>CPU</dt><dd>{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)</dd><dt>Memory available / total</dt><dd>{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}</dd><dt>Game storage available / total</dt><dd>{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}</dd><dt>Docker</dt><dd>{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}</dd><dt>Current operation</dt><dd>{host.inventory.currentOperation ?? 'None reported'}</dd>
-        </dl></section><section class="panel"><h2>Host configuration</h2>{#if canManage}<form onsubmit={save}><fieldset disabled={busy}>
+        </dl></section><section class="panel"><h2>Host configuration</h2>{#if canManage}<form onsubmit={save}><fieldset disabled={busy || cleaning}>
             <label>Friendly name<input required maxlength="200" bind:value={name} /></label><div class="fields"><label>First game port<input type="number" min="1" max="65535" required bind:value={portMin} /></label><label>Last game port<input type="number" min={portMin} max="65535" required bind:value={portMax} /></label></div><label>Local bind address<input required bind:value={bindAddress} /></label><label>Public / NAT address<input bind:value={publicAddress} placeholder="Optional public hostname or address" /></label><label>Minimum free game storage (GiB)<input type="number" min="0" step="0.1" required bind:value={minimumFreeGB} /></label><Button type="submit" disabled={busy}>Save settings</Button>
         </fieldset></form>{:else}<dl><dt>Friendly name</dt><dd>{host.name}</dd><dt>Game ports</dt><dd>{host.portMin}–{host.portMax}</dd><dt>Local bind address</dt><dd>{host.bindAddress}</dd><dt>Public / NAT address</dt><dd>{host.publicAddress || 'Not configured'}</dd><dt>Minimum free storage</dt><dd>{bytes(host.minimumFreeBytes)}</dd></dl><p>Host configuration is managed by administrators.</p>{/if}<p>Public addressing is separate from local port reservations. Configure your firewall and NAT for the game-port range.</p></section></div>
         <div class="grid"><section class="panel"><h2>Game files and runtime</h2><p><a href="/game-files">Manage game-file libraries →</a></p>{#if host.inventory.libraries?.length}<ul>{#each host.inventory.libraries as library}<li>{library.id}: {library.state}{library.progress !== undefined ? ` (${library.progress}%)` : ''}</li>{/each}</ul>{:else}<p>No cached game libraries reported.</p>{/if}
-            {#if canManage}<form onsubmit={(event) => { event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}><fieldset disabled={busy}><label>Plutainer image<input required bind:value={runtimeImage} /></label><Button type="submit" disabled={busy || !host.connected}>Prepare runtime</Button></fieldset></form>{/if}<p>Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported'}</p>
-        </section><section class="panel"><h2>Agent access</h2><p>Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.</p>{#if canManage}<div class="actions"><Button disabled={busy} onclick={() => action('/enrollment', 'POST')}>Generate installer</Button><Button disabled={busy || !host.credentialActive || !host.connected} onclick={() => action('/credentials', 'POST', undefined, 'Credential rotation queued.')}>Rotate credential</Button><Button disabled={busy || !host.credentialActive} onclick={() => action('/credentials', 'DELETE', undefined, 'Host credentials revoked. Generate a new installer to enroll again.')}>Revoke credential</Button></div>
+            {#if canManage}<form onsubmit={(event) => { event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}><fieldset disabled={busy || cleaning}><label>Plutainer image<input required bind:value={runtimeImage} /></label><Button type="submit" disabled={busy || !host.connected}>Prepare runtime</Button></fieldset></form>{/if}<p>Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported'}</p>
+        </section><section class="panel"><h2>Agent access</h2><p>Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.</p>{#if canManage}<div class="actions"><Button disabled={busy || (cleaning && !host.decommission?.retryable)} onclick={() => action('/enrollment', 'POST')}>Generate installer</Button><Button disabled={busy || cleaning || !host.credentialActive || !host.connected} onclick={() => action('/credentials', 'POST', undefined, 'Credential rotation queued.')}>Rotate credential</Button><Button disabled={busy || cleaning || !host.credentialActive} onclick={() => action('/credentials', 'DELETE', undefined, 'Host credentials revoked. Generate a new installer to enroll again.')}>Revoke credential</Button></div>
             {#if enrollment}<p>Expires {new Date(enrollment.expiresAt).toLocaleString()}</p><pre>{enrollment.command}</pre>{/if}{:else}<p>Administrators manage enrollment and agent credentials.</p>{/if}
         </section></div>
         <section class="panel"><h2>Operation history</h2>{#if !operations.length}<p>No remote operations yet.</p>{:else}<div class="table-wrap"><table><thead><tr><th>Operation</th><th>State</th><th>Updated</th><th>Result</th></tr></thead><tbody>{#each operations as operation}<tr><td>{operation.kind}<small>{operation.id}</small></td><td>{operation.state}</td><td>{new Date(operation.updatedAt).toLocaleString()}</td><td>{operation.error ?? '—'}</td></tr>{/each}</tbody></table></div>{/if}</section>
     {/if}
 </section>
-<DeleteHostDialog bind:open={deleteOpen} {host} ondeleted={() => goto('/hosts')} />
+<DeleteHostDialog bind:open={deleteOpen} {host} ondeleted={async (result) => { if (result.deleted) await goto('/hosts'); else { enrollment = null; await load(); } }} />
 
 <style>
     .page { display:grid; gap:24px; } .panel { border:1px solid var(--border-strong); border-radius:var(--radius-md); background:var(--surface); padding:24px; min-width:0; } h2 { margin:0 0 16px; font-size:var(--text-md); text-transform:capitalize; } p, li, dd, dt, span { font-size:var(--text-sm); line-height:1.65; } p,li,dt,span { color:var(--text-secondary); } .grid { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:24px; } .row,.actions,.fields { display:flex; gap:12px; flex-wrap:wrap; } .row { justify-content:space-between; } .actions { margin-top:20px; } dl { display:grid; grid-template-columns:1fr 1.3fr; gap:12px; margin:0; } dd { margin:0; overflow-wrap:anywhere; } fieldset { display:grid; gap:16px; border:0; padding:0; margin:0; } label { display:grid; gap:8px; font-size:var(--text-sm); } input { width:100%; min-width:0; padding:10px; color:var(--text-primary); background:var(--surface-sunken); border:1px solid var(--border-control); border-radius:var(--radius-sm); } .fields>* { flex:1; min-width:120px; } pre { white-space:pre-wrap; overflow-wrap:anywhere; padding:16px; background:var(--surface-sunken); font-size:var(--text-xs); } .table-wrap { overflow:auto; } table { width:100%; border-collapse:collapse; text-align:left; font-size:var(--text-sm); } td,th { padding:12px; border-bottom:1px solid var(--border-subtle); } small { display:block; color:var(--text-secondary); font-size:var(--text-xs); margin-top:4px; } @media(max-width:900px) { .grid { grid-template-columns:1fr; } }

static/agent/ec-agent.py

@@ -29,10 +29,17 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.0.0"
+VERSION = "1.1.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
+AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
+AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
+AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service")
+UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
+UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service")
+UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
+ROOT_MARKER = ".electric-cherry.json"
 
 
 class AgentError(Exception):
@@ -47,6 +54,97 @@ class Rejected(AgentError):
     """A definitive precondition failure, before any destructive action."""
 
 
+def systemctl(*args):
+    try:
+        result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
+    except (OSError, subprocess.TimeoutExpired):
+        raise AgentError("Host service management is unavailable") from None
+    if result.returncode:
+        raise AgentError("Host service management did not complete")
+
+
+def protected_path(path, missing=False):
+    """Deletion authority must never come from a runtime-writable path."""
+    path = Path(path)
+    for current in (*reversed(path.parents), path):
+        try:
+            info = current.lstat()
+        except FileNotFoundError:
+            if missing:
+                continue
+            raise AgentError("Host uninstall files are unavailable; reinstall the agent") from None
+        if stat_module.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+            raise AgentError("Host uninstall paths must be owned and writable only by root")
+    return path
+
+
+def protected_file(path, missing=False):
+    path = protected_path(path, missing=missing)
+    try:
+        if not stat_module.S_ISREG(path.lstat().st_mode):
+            raise AgentError("Host uninstall metadata must be a regular file")
+    except FileNotFoundError:
+        if not missing:
+            raise
+    return path
+
+
+def sync_parent(path):
+    descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def uninstall_root(value, identity=None):
+    """Accept only a canonical, dedicated directory with no nested mounts."""
+    if not isinstance(value, str) or not re.fullmatch(r"/[A-Za-z0-9/_.-]+", value):
+        raise AgentError("Host data directory is unsafe to remove")
+    root = Path(value)
+    forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"}
+    if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden:
+        raise AgentError("Host data directory is unsafe to remove")
+    for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT):
+        if installed == root or root in installed.parents:
+            raise AgentError("Host data directory overlaps the agent installation")
+    protected_path(root, missing=True)
+    if root.exists():
+        info = root.stat()
+        if not stat_module.S_ISDIR(info.st_mode):
+            raise AgentError("Host data directory is not a directory")
+        if identity and [info.st_dev, info.st_ino] != identity:
+            raise AgentError("Host data directory changed since deletion was authorized")
+    try:
+        # ProtectSystem/ReadWritePaths give the running agent a private bind
+        # mount at its data directory. Inspect the host systemd namespace,
+        # otherwise every legitimate installation appears to be a mount root.
+        mount_lines = Path("/proc/1/mountinfo").read_text().splitlines()
+    except FileNotFoundError:
+        mount_lines = []  # Non-Linux tests; the installed service requires Linux.
+    for line in mount_lines:
+        fields = line.split()
+        if len(fields) < 5:
+            continue
+        mount = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), fields[4])
+        if mount == value or mount.startswith(value + "/"):
+            raise AgentError("Unmount filesystems inside the host data directory before deletion")
+    return root
+
+
+def uninstall_ownership(config):
+    root = uninstall_root(config.get("root"))
+    marker = protected_file(root / ROOT_MARKER)
+    try:
+        saved = json.loads(marker.read_text())
+    except (OSError, ValueError):
+        raise AgentError("Host data ownership is unavailable; reinstall the agent") from None
+    expected = {"version": 1, "root": str(root), "hostId": config.get("hostId"), "controller": config.get("controller", "").rstrip("/")}
+    if saved != expected or not expected["hostId"]:
+        raise AgentError("Host data ownership does not match this enrollment")
+    return root
+
+
 def digest(data):
     return hashlib.sha256(data).hexdigest()
 
@@ -275,7 +373,7 @@ class Docker:
             raise AgentError("Docker command unavailable or timed out") from None
         if result.returncode:
             # Never include arbitrary Docker stderr: create errors may echo secrets.
-            if tolerate_missing and (b"No such object" in (result.stderr or b"") or b"No such container" in (result.stderr or b"")):
+            if tolerate_missing and any(message in (result.stderr or b"") for message in (b"No such object", b"No such container", b"No such image")):
                 return None
             raise AgentError("Docker rejected the requested operation")
         return result.stdout.decode("utf-8", "replace")
@@ -285,6 +383,188 @@ class Docker:
         return json.loads(raw)[0] if raw is not None else None
 
 
+class Uninstaller:
+    """A separate systemd service owns cleanup after stopping the host agent.
+
+    Its root-only request and executable stay on disk until the controller
+    acknowledges data removal. Completion is reported only after removing those
+    final files. A lost final response can be retried from memory; a power loss
+    in that last window deliberately leaves controller status unconfirmed.
+    """
+
+    def __init__(self, request_path, docker=None):
+        if Path(request_path) != UNINSTALL_REQUEST:
+            raise AgentError("Invalid host uninstall request location")
+        protected_file(UNINSTALL_REQUEST)
+        self.job = json.loads(UNINSTALL_REQUEST.read_text())
+        self.docker = docker or Docker()
+        self.root = uninstall_root(self.job.get("root"), self.job.get("rootIdentity"))
+        host_id = identifier(self.job.get("hostId"))
+        operation_id = identifier(self.job.get("operationId"))
+        parsed = urllib.parse.urlsplit(self.job.get("controller", ""))
+        if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise AgentError("Host uninstall controller must use HTTPS")
+        if self.job.get("receiptUrl") != "/api/agent/decommission/" + operation_id or not isinstance(self.job.get("receiptToken"), str) or not self.job["receiptToken"]:
+            raise AgentError("Invalid host uninstall receipt")
+        if not isinstance(self.job.get("rootIdentity"), list) or len(self.job["rootIdentity"]) != 2 or not all(isinstance(value, int) and not isinstance(value, bool) for value in self.job["rootIdentity"]):
+            raise AgentError("Invalid host data ownership proof")
+        if self.root.exists():
+            # The authorized device/inode is persisted before deletion so a
+            # crash halfway through rmtree does not require a deleted marker.
+            if not self.job.get("removalStarted"):
+                uninstall_ownership({"root": str(self.root), "controller": self.job["controller"], "hostId": host_id})
+
+    def save(self, **changes):
+        self.job.update(changes)
+        atomic_json(UNINSTALL_REQUEST, self.job)
+
+    def receipt(self, stage, error=None):
+        payload = {"stage": stage}
+        if error:
+            payload["error"] = error
+        request = urllib.request.Request(self.job["controller"].rstrip("/") + self.job["receiptUrl"], data=json.dumps(payload).encode(), headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.job["receiptToken"], "User-Agent": "electric-cherry-agent/" + VERSION}, method="POST")
+        class NoRedirect(urllib.request.HTTPRedirectHandler):
+            def redirect_request(self, req, fp, code, msg, hdrs, newurl):
+                raise AgentError("Controller redirects are not allowed")
+        try:
+            with urllib.request.build_opener(NoRedirect()).open(request, timeout=35) as response:
+                if json.load(response).get("acknowledged") is not True:
+                    raise AgentError("Controller has not acknowledged host removal")
+        except (OSError, ValueError):
+            raise AgentError("Host removal receipt is waiting for controller contact") from None
+
+    def containers(self):
+        ids = self.docker.call("container", "ls", "--all", "--quiet", "--no-trunc").splitlines()
+        values = []
+        for container_id in ids:
+            info = self.docker.inspect(container_id)
+            if info:
+                values.append(info)
+        return values
+
+    def owned_container(self, info):
+        labels = info.get("Config", {}).get("Labels", {}) or {}
+        if labels.get(LABEL + ".managed") != "true":
+            return False
+        if labels.get(LABEL + ".host") not in (None, self.job["hostId"]):
+            return False
+        try:
+            instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", "")))
+        except (AgentError, ValueError, TypeError):
+            return False
+        expected = str(self.root / "instances" / instance_id / str(generation) / "app")
+        # Legacy agents did not label the host; its exact managed bind mount
+        # proves ownership without touching another installation's containers.
+        return any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", []))
+
+    def uses_root(self, info):
+        for mount in info.get("Mounts", []):
+            if mount.get("Type") != "bind":
+                continue
+            source = Path(mount.get("Source", "/")).resolve()
+            if source == self.root or self.root in source.parents:
+                return True
+        return False
+
+    def remove_docker_resources(self):
+        containers = self.containers()
+        if any(self.uses_root(info) and not self.owned_container(info) for info in containers):
+            raise AgentError("An unrelated container uses the host data directory; detach it before retrying deletion")
+        owned = [info for info in containers if self.owned_container(info)]
+        if any(info.get("State", {}).get("Running") is not False or info.get("State", {}).get("Status") not in ("created", "exited", "dead") for info in owned):
+            raise AgentError("A managed game container is still running or its state is unknown; retire it before deleting this host")
+        for info in owned:
+            # Do not force removal: Docker must also reject a container that
+            # starts again after inspection rather than interrupting players.
+            self.docker.call("container", "rm", "--volumes", info["Id"], timeout=120, tolerate_missing=True)
+        remaining = self.containers()
+        if any(self.uses_root(info) for info in remaining):
+            raise AgentError("Containers still use the host data directory")
+        # Never prune Docker globally. Only remove this agent's recorded image
+        # references, and preserve images used by any unrelated container.
+        used_images = {info.get("Image") for info in remaining}
+        images = self.job.get("runtimeImages", {})
+        if not isinstance(images, dict):
+            raise AgentError("Managed runtime image ownership is unavailable")
+        for reference, expected_id in images.items():
+            if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,511}", reference) or not isinstance(expected_id, str):
+                raise AgentError("Managed runtime image ownership is invalid")
+            raw = self.docker.call("image", "inspect", reference, tolerate_missing=True)
+            if raw is None:
+                continue
+            found = json.loads(raw)
+            if not found or found[0].get("Id") != expected_id or expected_id in used_images:
+                continue
+            self.docker.call("image", "rm", reference, tolerate_missing=True)
+
+    def remove_data(self):
+        protected_path(AGENT_CONFIG.parent, missing=True)
+        if not self.job.get("agentStopped"):
+            systemctl("disable", "--now", AGENT_UNIT.name)
+            self.save(agentStopped=True)
+        self.remove_docker_resources()
+        root = uninstall_root(str(self.root), self.job["rootIdentity"])
+        # This checkpoint also authorizes safe continuation if a power loss
+        # removes the ownership marker before the rest of the tree.
+        self.save(removalStarted=True)
+        if root.exists():
+            if not shutil.rmtree.avoids_symlink_attacks:
+                raise AgentError("This platform cannot safely remove managed host files")
+            shutil.rmtree(root)
+            sync_parent(root)
+        for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT):
+            protected_file(path, missing=True)
+            path.unlink(missing_ok=True)
+            sync_parent(path)
+        systemctl("daemon-reload")
+        self.save(dataRemoved=True)
+
+    def remove_helper(self):
+        # Disable does not stop this process: it must outlive its own unit and
+        # executable to deliver the final receipt after the files are gone.
+        protected_file(UNINSTALL_UNIT, missing=True)
+        if UNINSTALL_UNIT.exists():
+            systemctl("disable", UNINSTALL_UNIT.name)
+            UNINSTALL_UNIT.unlink()
+        sync_parent(UNINSTALL_UNIT)
+        systemctl("daemon-reload")
+        protected_file(UNINSTALL_EXECUTABLE, missing=True)
+        UNINSTALL_EXECUTABLE.unlink(missing_ok=True)
+        sync_parent(UNINSTALL_EXECUTABLE)
+        protected_path(UNINSTALL_REQUEST.parent, missing=True)
+        if UNINSTALL_REQUEST.parent.exists():
+            shutil.rmtree(UNINSTALL_REQUEST.parent)
+        sync_parent(UNINSTALL_REQUEST.parent)
+
+    def run(self):
+        if not self.job.get("dataRemoved"):
+            try:
+                self.remove_data()
+            except (AgentError, OSError, ValueError, KeyError) as error:
+                with contextlib.suppress(AgentError):
+                    self.receipt("failed", str(error) if isinstance(error, AgentError) else "Host cleanup could not finish. Check the uninstall service and resolve filesystem errors; cleanup will retry.")
+                raise
+        self.receipt("cleaned")
+        while True:
+            try:
+                self.remove_helper()
+                break
+            except (AgentError, OSError, ValueError):
+                # Once its unit/executable has been unlinked systemd can no
+                # longer restart this worker. Finish idempotently in memory.
+                with contextlib.suppress(AgentError):
+                    self.receipt("failed", "Host files were removed, but removing the uninstall service is still retrying")
+                time.sleep(15)
+        # The controller never marks a host deleted on the earlier receipt.
+        # No credentials or retry state are left on disk at this point.
+        while True:
+            try:
+                self.receipt("completed")
+                return
+            except AgentError:
+                time.sleep(15)
+
+
 class Agent:
     def __init__(self, config, docker=None):
         self.config = config
@@ -300,6 +580,43 @@ class Agent:
         self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
         self.executing_id = None
 
+    def can_uninstall(self):
+        try:
+            if Path(self.config_path) != AGENT_CONFIG:
+                return False
+            uninstall_ownership(self.config)
+            protected_file(UNINSTALL_EXECUTABLE)
+            protected_file(UNINSTALL_UNIT)
+            return True
+        except (AgentError, OSError, ValueError):
+            return False
+
+    def decommission(self, operation):
+        payload = operation.get("payload", {})
+        if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]:
+            raise Rejected("Host removal identity does not match this agent")
+        if payload.get("receiptUrl") != "/api/agent/decommission/" + identifier(operation["id"]) or not isinstance(payload.get("receiptToken"), str) or not payload["receiptToken"]:
+            raise Rejected("Invalid host removal receipt")
+        if not self.can_uninstall():
+            raise Rejected("Reinstall the agent to enable safe removal of its dedicated host files")
+        root = uninstall_ownership(self.config)
+        info = root.stat()
+        protected_file(UNINSTALL_REQUEST, missing=True)
+        aliases_path = root / "agent" / "runtime-images.json"
+        protected_file(aliases_path, missing=True)
+        aliases = json.loads(aliases_path.read_text()) if aliases_path.exists() else {}
+        job = {"hostId": payload["hostId"], "operationId": operation["id"], "receiptUrl": payload["receiptUrl"], "receiptToken": payload["receiptToken"], "controller": self.config["controller"].rstrip("/"), "root": str(root), "rootIdentity": [info.st_dev, info.st_ino], "runtimeImages": aliases}
+        if UNINSTALL_REQUEST.exists():
+            previous = json.loads(UNINSTALL_REQUEST.read_text())
+            if any(previous.get(key) != job[key] for key in ("hostId", "operationId", "receiptUrl", "receiptToken", "controller", "root", "rootIdentity")):
+                raise Rejected("A different host removal is already pending")
+        self.checkpoint({"phase": "uninstall-service"})
+        if not UNINSTALL_REQUEST.exists():
+            atomic_json(UNINSTALL_REQUEST, job)
+        systemctl("enable", "--now", "--no-block", UNINSTALL_UNIT.name)
+        self.stopping.set()
+        return {"decommissioning": True, "message": "The uninstall service will report completion after host files and the agent are removed"}
+
     def instance_dir(self, instance_id, generation):
         instance_id, generation = instance_identity(instance_id, generation)
         return self.root / "instances" / instance_id / str(generation)
@@ -416,7 +733,7 @@ class Agent:
             for line in Path("/etc/os-release").read_text().splitlines():
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
-        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
+        return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
         health = self.inspect(instance_id, generation)
@@ -525,6 +842,8 @@ class Agent:
         name = self.container_name(instance_id, generation)
         spec_sha = digest(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode())
         args = ["container", "create", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)]
+        if self.config.get("hostId"):
+            args.extend(["--label", LABEL + ".host=" + identifier(self.config["hostId"])])
         bind_arg = "[" + bind + "]" if ":" in bind else bind
         for protocol in ("udp", "tcp"):
             args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
@@ -923,6 +1242,8 @@ class Agent:
         kind, payload = operation.get("kind"), operation.get("payload", {})
         if not isinstance(kind, str) or not isinstance(payload, dict):
             raise AgentError("Invalid operation")
+        if kind == "host.decommission":
+            return self.decommission(operation)
         if kind == "credentials.rotate":
             pending = "ec_host_" + base64.urlsafe_b64encode(os.urandom(32)).rstrip(b"=").decode()
             self.config["pendingCredential"] = pending
@@ -997,7 +1318,7 @@ class Agent:
             return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message)
         finally:
             self.executing_id = None
-        return self.journal.update(operation["id"], "completed", result)
+        return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result)
 
     def recover_credential(self):
         pending = self.config.get("pendingCredential")
@@ -1024,6 +1345,13 @@ class Agent:
         instance_id, generation = operation.get("instanceId"), operation.get("generation")
         checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
         try:
+            if kind == "host.decommission":
+                self.executing_id = operation["id"]
+                try:
+                    result = self.decommission(operation)
+                    return self.journal.update(operation["id"], "running", result)
+                finally:
+                    self.executing_id = None
             if kind == "credentials.rotate":
                 if not self.config.get("pendingCredential") or self.recover_credential():
                     return self.journal.update(operation["id"], "completed", {"rotated": True, "reconciled": True})
@@ -1183,8 +1511,14 @@ def main():
     parser.add_argument("--token-file")
     parser.add_argument("--root", default="/var/lib/electric-cherry")
     parser.add_argument("--force-enroll", action="store_true")
+    parser.add_argument("--uninstall", metavar="REQUEST", help="Run the installed host removal service")
     parser.add_argument("--version", action="version", version=VERSION)
     args = parser.parse_args()
+    if args.uninstall:
+        if os.geteuid() != 0 or args.enroll:
+            raise AgentError("Host uninstall must run as the dedicated root service")
+        Uninstaller(args.uninstall).run()
+        return
     if args.enroll:
         if not args.controller or not args.token_file:
             parser.error("Enrollment needs --controller and --token-file")

static/agent/install.sh

@@ -21,7 +21,7 @@ done
 case "$controller" in https://*) ;; *) echo '--controller must be an HTTPS URL.' >&2; exit 2 ;; esac
 [ -n "$enrollment_token" ] || { echo '--token is required.' >&2; exit 2; }
 case "$data_dir" in /*) ;; *) echo '--data-dir must be absolute.' >&2; exit 2 ;; esac
-case "$data_dir" in *[!a-zA-Z0-9/_.-]*|/|/etc|/usr|/var|/var/lib) echo 'Choose a dedicated data directory.' >&2; exit 2 ;; esac
+case "$data_dir" in *[!a-zA-Z0-9/_.-]*|*/../*|*/./*|*/..|*/.|*//*|*/) echo 'Choose a canonical, dedicated data directory.' >&2; exit 2 ;; esac
 [ ! -L "$data_dir" ] || { echo 'Data directory must not be a symlink.' >&2; exit 2; }
 . /etc/os-release
 [ "${ID:-}" = debian ] || { echo 'Supported platform: Debian 12 or 13.' >&2; exit 1; }
@@ -32,13 +32,82 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; }
 export DEBIAN_FRONTEND=noninteractive
 apt-get update -qq
 apt-get install -y --no-install-recommends ca-certificates curl python3
-python3 - "$controller" "$data_dir" <<'PY'
-import json, pathlib, sys
-config = pathlib.Path('/etc/electric-cherry/agent.json')
-if config.exists():
-    saved = json.loads(config.read_text())
-    if saved.get('controller', '').rstrip('/') != sys.argv[1].rstrip('/') or pathlib.Path(saved.get('root', '')).resolve() != pathlib.Path(sys.argv[2]).resolve():
+python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY'
+# INSTALLATION_PREFLIGHT: this runs before touching managed directories or services.
+import json, pathlib, re, stat, sys
+
+controller, raw_root, config_name = sys.argv[1:]
+root = pathlib.Path(raw_root)
+config = pathlib.Path(config_name)
+reserved = {
+    '/', '/bin', '/boot', '/dev', '/etc', '/home', '/lib', '/lib64',
+    '/media', '/mnt', '/opt', '/proc', '/root', '/run', '/sbin', '/srv',
+    '/sys', '/tmp', '/usr', '/usr/local', '/usr/local/bin', '/usr/local/lib',
+    '/usr/local/libexec', '/var', '/var/cache', '/var/lib', '/var/log', '/var/tmp',
+}
+if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root)
+        or str(root) != raw_root or any(part in ('.', '..') for part in raw_root.split('/')[1:])
+        or len(root.parts) < 3 or raw_root in reserved or config.parent == root or config.parent in root.parents):
+    raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.')
+installed_paths = [config,
+    pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'),
+    pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'),
+    pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service')]
+if any(root == path or root in path.parents for path in installed_paths):
+    raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.')
+
+def trusted_directories(path):
+    for current in [*reversed(path.parents), path]:
+        try:
+            info = current.lstat()
+        except FileNotFoundError:
+            continue
+        if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+            raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.')
+
+def trusted_json(path):
+    info = path.lstat()
+    if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
+        raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.')
+    value = json.loads(path.read_text())
+    if not isinstance(value, dict):
+        raise SystemExit('Existing installation metadata is invalid.')
+    return value
+
+trusted_directories(root)
+trusted_directories(config.parent)
+mountinfo = pathlib.Path('/proc/1/mountinfo')
+if mountinfo.exists():
+    for line in mountinfo.read_text().splitlines():
+        fields = line.split()
+        if len(fields) < 5:
+            continue
+        mount = re.sub(r'\\([0-7]{3})', lambda match: chr(int(match[1], 8)), fields[4])
+        if mount == raw_root or mount.startswith(raw_root + '/'):
+            raise SystemExit('Use a dedicated subdirectory of the mounted disk, with no filesystems mounted inside it, so host deletion can remove the entire data directory.')
+request = config.with_name('decommission.json')
+if request.exists() or request.is_symlink():
+    raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.')
+saved = None
+if config.exists() or config.is_symlink():
+    saved = trusted_json(config)
+    if saved.get('controller', '').rstrip('/') != controller.rstrip('/') or saved.get('root') != raw_root:
         raise SystemExit('Existing enrollment uses a different controller or data directory. Preserve the existing installation and migrate it explicitly.')
+    if not saved.get('hostId') or not saved.get('credential'):
+        raise SystemExit('Existing enrollment is incomplete. Repair its credentials before upgrading.')
+marker = root / '.electric-cherry.json'
+if marker.exists() or marker.is_symlink():
+    owned = trusted_json(marker)
+    if not saved or owned != {'version': 1, 'root': raw_root, 'hostId': saved['hostId'], 'controller': controller.rstrip('/')}:
+        raise SystemExit('Data directory ownership does not match this enrollment. Preserve it and migrate explicitly.')
+elif root.exists() and any(root.iterdir()):
+    # Upgrade installations predating ownership markers only when the entire
+    # root has the known layout. Never adopt a directory of unrelated files.
+    if not saved or any(path.name not in {'agent', 'instances', 'libraries'} for path in root.iterdir()):
+        raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.')
+for directory in ('agent', 'instances', 'libraries'):
+    trusted_directories(root / directory)
 PY
 if ! command -v docker >/dev/null; then
     if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then
@@ -76,9 +145,36 @@ install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-
     install -d -m 0755 /usr/local/libexec
     install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py
 }
+# A separate worker survives stopping/removing the agent service. Its only
+# entry point is a fixed, authenticated deletion request prepared by the agent.
+install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py
 printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token"
 unset enrollment_token
 python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"}
+python3 - "$data_dir" /etc/electric-cherry/agent.json <<'PY'
+# INSTALLATION_OWNERSHIP: bind deletion authority to this exact enrollment/root.
+import json, os, pathlib, sys, uuid
+root = pathlib.Path(sys.argv[1])
+config = json.loads(pathlib.Path(sys.argv[2]).read_text())
+if config.get('root') != str(root) or not config.get('hostId') or not config.get('controller'):
+    raise SystemExit('Enrollment did not establish ownership of this data directory.')
+marker = root / '.electric-cherry.json'
+temporary = root / ('.electric-cherry.' + uuid.uuid4().hex + '.tmp')
+try:
+    with temporary.open('x') as stream:
+        os.chmod(temporary, 0o600)
+        json.dump({'version': 1, 'root': str(root), 'hostId': config['hostId'], 'controller': config['controller'].rstrip('/')}, stream)
+        stream.flush()
+        os.fsync(stream.fileno())
+    os.replace(temporary, marker)
+    directory_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(directory_fd)
+    finally:
+        os.close(directory_fd)
+finally:
+    temporary.unlink(missing_ok=True)
+PY
 cat > /etc/systemd/system/electric-cherry-agent.service <<EOF
 [Unit]
 Description=Electric Cherry Host Agent
@@ -99,6 +195,23 @@ ProtectHome=true
 PrivateTmp=true
 ReadWritePaths=$data_dir /etc/electric-cherry
 
+[Install]
+WantedBy=multi-user.target
+EOF
+cat > /etc/systemd/system/electric-cherry-uninstall.service <<'EOF'
+[Unit]
+Description=Remove an Electric Cherry Host Installation
+After=network-online.target docker.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-uninstall.py --uninstall /etc/electric-cherry/decommission.json
+Restart=on-failure
+RestartSec=15
+TimeoutStartSec=0
+UMask=0077
+
 [Install]
 WantedBy=multi-user.target
 EOF

static/agent/manifest.json

@@ -1,6 +1,6 @@
 {
-  "version": "1.0.0",
+  "version": "1.1.0",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "2b6e6c2cc4c949e8f980c147aa6b8725844b7c320d7627e0328570d23c985a43"
+  "sha256": "67046a3d81f251e148f2323cade255ff584a622750a8c05ee27f96c70e09fcc7"
 }

tests/unit/auth-access.test.ts

@@ -92,6 +92,7 @@ describe('authentication and machine boundaries', () => {
     it.each([
         ['/api/agent/enroll', 'POST'], ['/api/agent/heartbeat', 'POST'], ['/api/agent/operations', 'GET'],
         ['/api/agent/operations/[id]', 'POST'], ['/api/agent/libraries/[id]/files/[...path]', 'GET'],
+        ['/api/agent/decommission/[id]', 'POST'],
         ['/api/agent/imports/[id]/files/[...path]', 'PUT'], ['/api/agent/imports/[id]/complete', 'POST'],
         ['/api/agent/install.sh', 'GET'], ['/api/agent/release/[file]', 'GET'], ['/api/telemetry', 'POST']
     ])('preserves endpoint-specific machine authentication for %s', (route, method) => {

tests/unit/host-deletion.test.ts

@@ -5,7 +5,7 @@ import { join } from 'node:path';
 import { newFixture, type Fixture } from '../helpers/fixture.js';
 
 let fixture: Fixture;
-const inventory = { hostname: 'games-1', protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3 };
+const inventory = { hostname: 'games-1', protocolVersion: 1, capabilities: ['host.decommission'], dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3 };
 beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
 afterEach(async () => { vi.unstubAllEnvs(); await fixture.cleanup(); });
 
@@ -16,6 +16,16 @@ async function enrolled() {
     return { ...hosts, host, credentials };
 }
 
+async function finishCleanup(hosts: Awaited<ReturnType<typeof enrolled>>) {
+    const result = hosts.deleteHost(hosts.host.id);
+    const operation = hosts.getHostOperation(result.operationId!)!;
+    hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
+    const token = operation.payload.receiptToken as string;
+    hosts.receiveDecommissionReceipt(operation.id, token, { stage: 'cleaned' });
+    hosts.receiveDecommissionReceipt(operation.id, token, { stage: 'completed' });
+    return operation;
+}
+
 async function instance(hostId: string, retired = false) {
     const { db } = await import('../../src/lib/server/db/index.js');
     db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at,retired_at)
@@ -46,7 +56,7 @@ describe('host deletion', () => {
         expect(() => hosts.deleteHost(host.id)).toThrow(expect.objectContaining({ status: 404 }));
     });
 
-    it('deletes an enrolled empty host, revokes credentials, and preserves files and history', async () => {
+    it('waits for host cleanup before revoking credentials and retains controller libraries and history', async () => {
         const hosts = await enrolled();
         const other = hosts.createHost({ name: 'Keep' });
         const otherCredentials = hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory);
@@ -64,14 +74,18 @@ describe('host deletion', () => {
             VALUES('version','library','1','','[]','ready',?,?,?,'t5_game_files')`).run(new Date().toISOString(), file, hosts.host.id);
         db().prepare("INSERT INTO host_library_caches(host_id,version_id,status,updated_at) VALUES(?,'version','ready',?)").run(hosts.host.id, new Date().toISOString());
 
-        hosts.deleteHost(hosts.host.id);
+        const queued = hosts.deleteHost(hosts.host.id);
+        expect(queued.deleted).toBe(false);
+        expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('queued');
+        expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
+        await finishCleanup(hosts);
 
         expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull();
         expect(hosts.authenticateCredential(otherCredentials.credential)?.id).toBe(other.id);
         expect(() => hosts.enrollHost(enrollment.token, inventory)).toThrow('invalid or expired');
         expect(hosts.listHosts().map(host => host.id)).toEqual([other.id]);
         expect(hosts.getHostOperation(completed.id)).toMatchObject({ state: 'completed', result: { hostname: 'games-1' } });
-        expect(hosts.listHostOperations(hosts.host.id)).toHaveLength(1);
+        expect(hosts.listHostOperations(hosts.host.id)).toHaveLength(2);
         expect(db().prepare('SELECT mode,deleted_at FROM ec_hosts WHERE id=?').get(hosts.host.id)).toEqual({ mode: 'disabled', deleted_at: expect.any(String) });
         expect(db().prepare('SELECT revoked_at FROM ec_host_credentials WHERE host_id=?').get(hosts.host.id)).toEqual({ revoked_at: expect.any(String) });
         expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({ count: 1 });
@@ -112,7 +126,7 @@ describe('host deletion', () => {
         expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('reserved game port')] });
         expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
         hosts.releaseHostPorts('unresolved-instance');
-        hosts.deleteHost(hosts.host.id);
+        await finishCleanup(hosts);
         expect(hosts.getHost(hosts.host.id)).toBeNull();
     });
 
@@ -145,7 +159,8 @@ describe('host deletion', () => {
         expect(await blocked.json()).toMatchObject({ error: expect.any(String), blockers: [expect.stringContaining('delivered operation')] });
         hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' });
         const removed = route.DELETE({ request: new Request(url, { method: 'DELETE' }), params });
-        expect(removed.status).toBe(200); expect(await removed.json()).toEqual({ deleted: true });
+        expect(removed.status).toBe(202); expect(await removed.json()).toMatchObject({ deleted: false, operationId: expect.any(String) });
+        await finishCleanup(hosts);
         expect(route.DELETE({ request: new Request(url, { method: 'DELETE' }), params }).status).toBe(404);
     });
 
@@ -161,4 +176,88 @@ describe('host deletion', () => {
         }
         expect(hosts.getHost(hosts.host.id)).not.toBeNull();
     });
+
+    it('requires an uninstall-capable agent and preserves enrollment access for upgrades', async () => {
+        const hosts = await enrolled();
+        hosts.heartbeat(hosts.host.id, { ...inventory, capabilities: [] });
+        expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('Update this host agent')] });
+        expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
+        expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy();
+        expect(hosts.getHost(hosts.host.id)?.mode).toBe('active');
+    });
+
+    it('keeps offline cleanup pending and blocks settings, new work, enrollment and credential revocation', async () => {
+        const hosts = await enrolled();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2020-01-01T00:00:00Z', hosts.host.id);
+        const queued = hosts.deleteHost(hosts.host.id);
+        expect(hosts.deleteHost(hosts.host.id)).toEqual(queued);
+        expect(hosts.getHost(hosts.host.id)).toMatchObject({ connected: false, readiness: 'deleting', decommission: { stage: 'queued' } });
+        expect(hosts.pendingHostOperations(hosts.host.id)).toEqual([]);
+        expect(() => hosts.updateHost(hosts.host.id, { mode: 'disabled' })).toThrow('cleanup');
+        expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow('cleanup');
+        expect(() => hosts.reserveHostPort(hosts.host.id, 'late-instance')).toThrow('cleanup');
+        expect(hosts.hostPortAvailable(hosts.host.id, hosts.host.portMin)).toBe(false);
+        expect(() => hosts.createEnrollment(hosts.host.id)).toThrow('cleanup');
+        expect(() => hosts.revokeCredentials(hosts.host.id)).toThrow('cleanup');
+        hosts.heartbeat(hosts.host.id, inventory);
+        expect(hosts.pendingHostOperations(hosts.host.id).map(operation => operation.id)).toEqual([queued.operationId]);
+    });
+
+    it('requires scoped receipts in order and never treats an ordinary completion report or loss of contact as removal', async () => {
+        const hosts = await enrolled();
+        const queued = hosts.deleteHost(hosts.host.id);
+        const operation = hosts.getHostOperation(queued.operationId!)!;
+        const token = operation.payload.receiptToken as string;
+        const receipt = (stage: string, credential = token) => hosts.receiveDecommissionReceipt(operation.id, credential, { stage });
+        expect(() => receipt('cleaned')).toThrow(expect.objectContaining({ status: 409 }));
+        hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
+        expect(() => receipt('cleaned', hosts.credentials.credential)).toThrow(expect.objectContaining({ status: 401 }));
+        expect(() => receipt('completed')).toThrow(expect.objectContaining({ status: 409 }));
+        expect(() => hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' })).toThrow('cleanup worker');
+        receipt('cleaned');
+        expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('cleaned');
+        expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
+        const route = await import('../../src/routes/api/hosts/[id]/+server.js');
+        const detail = await route.GET({ request: new Request('https://ec.test/api/hosts/' + hosts.host.id), params: { id: hosts.host.id } }).text();
+        expect(detail).not.toContain(token);
+        receipt('completed');
+        expect(hosts.getHost(hosts.host.id)).toBeNull();
+        expect(receipt('completed')).toEqual({ acknowledged: true });
+        expect(receipt('cleaned')).toEqual({ acknowledged: true });
+        expect(hosts.getHostOperation(operation.id)?.state).toBe('completed');
+        expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull();
+    });
+
+    it('retains cleanup failures and retries only work rejected before mutation with a new scoped receipt', async () => {
+        const hosts = await enrolled();
+        const first = hosts.deleteHost(hosts.host.id);
+        hosts.reportHostOperation(hosts.host.id, first.operationId!, { state: 'failed', error: 'Ownership marker requires repair' });
+        expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ stage: 'failed', retryable: true });
+        expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy();
+        const second = hosts.deleteHost(hosts.host.id);
+        expect(second.operationId).not.toBe(first.operationId);
+        const operation = hosts.getHostOperation(second.operationId!)!;
+        hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
+        hosts.receiveDecommissionReceipt(operation.id, operation.payload.receiptToken as string, { stage: 'failed', error: 'Filesystem cleanup failed; retrying' });
+        expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ stage: 'failed', retryable: false, error: 'Filesystem cleanup failed; retrying' });
+        expect(hosts.deleteHost(hosts.host.id)).toEqual(second);
+        await finishCleanup(hosts);
+        expect(hosts.getHostOperation(first.operationId!)?.state).toBe('failed');
+    });
+
+    it('authenticates cleanup receipts independently after the agent credential has been revoked', async () => {
+        const hosts = await enrolled();
+        const queued = hosts.deleteHost(hosts.host.id);
+        const operation = hosts.getHostOperation(queued.operationId!)!;
+        hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
+        const { POST } = await import('../../src/routes/api/agent/decommission/[id]/+server.js');
+        const send = (stage: string, token: string) => POST({ request: new Request(`https://ec.test/api/agent/decommission/${operation.id}`, {
+            method: 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ stage })
+        }), params: { id: operation.id } });
+        expect((await send('cleaned', hosts.credentials.credential)).status).toBe(401);
+        expect((await send('cleaned', operation.payload.receiptToken as string)).status).toBe(200);
+        expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(200);
+        expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(200);
+    });
 });