From 020894b816782c8bb57a51637648791da1bf001f Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Sun, 4 Oct 2026 10:20:51 -0400 Subject: [PATCH] Remove host files and uninstall agent when deleting hosts --- agent/ec_agent.py | 342 +++++++++++++++++- agent/install.sh | 127 ++++++- agent/package_release.py | 2 +- agent/tests/test_installer.py | 185 ++++++++++ agent/tests/test_uninstall.py | 338 +++++++++++++++++ docs/authentication.md | 3 + docs/native-hosts.md | 53 ++- src/lib/hosts.ts | 20 +- src/lib/server/hosts/index.ts | 103 +++++- src/lib/server/hosts/schema.ts | 7 + src/lib/ui/components/DeleteHostDialog.svelte | 11 +- .../api/agent/decommission/[id]/+server.ts | 14 + src/routes/api/hosts/[id]/+server.ts | 2 +- src/routes/hosts/+page.svelte | 8 +- src/routes/hosts/[id]/+page.svelte | 18 +- static/agent/ec-agent.py | 342 +++++++++++++++++- static/agent/install.sh | 127 ++++++- static/agent/manifest.json | 4 +- tests/unit/auth-access.test.ts | 1 + tests/unit/host-deletion.test.ts | 111 +++++- 20 files changed, 1743 insertions(+), 75 deletions(-) create mode 100644 agent/tests/test_installer.py create mode 100644 agent/tests/test_uninstall.py create mode 100644 src/routes/api/agent/decommission/[id]/+server.ts diff --git a/agent/ec_agent.py b/agent/ec_agent.py index db8ab0c..02cd5c8 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -29,10 +29,17 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.0.0" +VERSION = "1.1.0" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$") +AGENT_CONFIG = Path("/etc/electric-cherry/agent.json") +AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py") +AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service") +UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py") +UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service") +UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json") +ROOT_MARKER = ".electric-cherry.json" class AgentError(Exception): @@ -47,6 +54,97 @@ class Rejected(AgentError): """A definitive precondition failure, before any destructive action.""" +def systemctl(*args): + try: + result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False) + except (OSError, subprocess.TimeoutExpired): + raise AgentError("Host service management is unavailable") from None + if result.returncode: + raise AgentError("Host service management did not complete") + + +def protected_path(path, missing=False): + """Deletion authority must never come from a runtime-writable path.""" + path = Path(path) + for current in (*reversed(path.parents), path): + try: + info = current.lstat() + except FileNotFoundError: + if missing: + continue + raise AgentError("Host uninstall files are unavailable; reinstall the agent") from None + if stat_module.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022: + raise AgentError("Host uninstall paths must be owned and writable only by root") + return path + + +def protected_file(path, missing=False): + path = protected_path(path, missing=missing) + try: + if not stat_module.S_ISREG(path.lstat().st_mode): + raise AgentError("Host uninstall metadata must be a regular file") + except FileNotFoundError: + if not missing: + raise + return path + + +def sync_parent(path): + descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def uninstall_root(value, identity=None): + """Accept only a canonical, dedicated directory with no nested mounts.""" + if not isinstance(value, str) or not re.fullmatch(r"/[A-Za-z0-9/_.-]+", value): + raise AgentError("Host data directory is unsafe to remove") + root = Path(value) + forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"} + if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden: + raise AgentError("Host data directory is unsafe to remove") + for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT): + if installed == root or root in installed.parents: + raise AgentError("Host data directory overlaps the agent installation") + protected_path(root, missing=True) + if root.exists(): + info = root.stat() + if not stat_module.S_ISDIR(info.st_mode): + raise AgentError("Host data directory is not a directory") + if identity and [info.st_dev, info.st_ino] != identity: + raise AgentError("Host data directory changed since deletion was authorized") + try: + # ProtectSystem/ReadWritePaths give the running agent a private bind + # mount at its data directory. Inspect the host systemd namespace, + # otherwise every legitimate installation appears to be a mount root. + mount_lines = Path("/proc/1/mountinfo").read_text().splitlines() + except FileNotFoundError: + mount_lines = [] # Non-Linux tests; the installed service requires Linux. + for line in mount_lines: + fields = line.split() + if len(fields) < 5: + continue + mount = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), fields[4]) + if mount == value or mount.startswith(value + "/"): + raise AgentError("Unmount filesystems inside the host data directory before deletion") + return root + + +def uninstall_ownership(config): + root = uninstall_root(config.get("root")) + marker = protected_file(root / ROOT_MARKER) + try: + saved = json.loads(marker.read_text()) + except (OSError, ValueError): + raise AgentError("Host data ownership is unavailable; reinstall the agent") from None + expected = {"version": 1, "root": str(root), "hostId": config.get("hostId"), "controller": config.get("controller", "").rstrip("/")} + if saved != expected or not expected["hostId"]: + raise AgentError("Host data ownership does not match this enrollment") + return root + + def digest(data): return hashlib.sha256(data).hexdigest() @@ -275,7 +373,7 @@ class Docker: raise AgentError("Docker command unavailable or timed out") from None if result.returncode: # Never include arbitrary Docker stderr: create errors may echo secrets. - if tolerate_missing and (b"No such object" in (result.stderr or b"") or b"No such container" in (result.stderr or b"")): + if tolerate_missing and any(message in (result.stderr or b"") for message in (b"No such object", b"No such container", b"No such image")): return None raise AgentError("Docker rejected the requested operation") return result.stdout.decode("utf-8", "replace") @@ -285,6 +383,188 @@ class Docker: return json.loads(raw)[0] if raw is not None else None +class Uninstaller: + """A separate systemd service owns cleanup after stopping the host agent. + + Its root-only request and executable stay on disk until the controller + acknowledges data removal. Completion is reported only after removing those + final files. A lost final response can be retried from memory; a power loss + in that last window deliberately leaves controller status unconfirmed. + """ + + def __init__(self, request_path, docker=None): + if Path(request_path) != UNINSTALL_REQUEST: + raise AgentError("Invalid host uninstall request location") + protected_file(UNINSTALL_REQUEST) + self.job = json.loads(UNINSTALL_REQUEST.read_text()) + self.docker = docker or Docker() + self.root = uninstall_root(self.job.get("root"), self.job.get("rootIdentity")) + host_id = identifier(self.job.get("hostId")) + operation_id = identifier(self.job.get("operationId")) + parsed = urllib.parse.urlsplit(self.job.get("controller", "")) + if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment: + raise AgentError("Host uninstall controller must use HTTPS") + if self.job.get("receiptUrl") != "/api/agent/decommission/" + operation_id or not isinstance(self.job.get("receiptToken"), str) or not self.job["receiptToken"]: + raise AgentError("Invalid host uninstall receipt") + if not isinstance(self.job.get("rootIdentity"), list) or len(self.job["rootIdentity"]) != 2 or not all(isinstance(value, int) and not isinstance(value, bool) for value in self.job["rootIdentity"]): + raise AgentError("Invalid host data ownership proof") + if self.root.exists(): + # The authorized device/inode is persisted before deletion so a + # crash halfway through rmtree does not require a deleted marker. + if not self.job.get("removalStarted"): + uninstall_ownership({"root": str(self.root), "controller": self.job["controller"], "hostId": host_id}) + + def save(self, **changes): + self.job.update(changes) + atomic_json(UNINSTALL_REQUEST, self.job) + + def receipt(self, stage, error=None): + payload = {"stage": stage} + if error: + payload["error"] = error + request = urllib.request.Request(self.job["controller"].rstrip("/") + self.job["receiptUrl"], data=json.dumps(payload).encode(), headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.job["receiptToken"], "User-Agent": "electric-cherry-agent/" + VERSION}, method="POST") + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, hdrs, newurl): + raise AgentError("Controller redirects are not allowed") + try: + with urllib.request.build_opener(NoRedirect()).open(request, timeout=35) as response: + if json.load(response).get("acknowledged") is not True: + raise AgentError("Controller has not acknowledged host removal") + except (OSError, ValueError): + raise AgentError("Host removal receipt is waiting for controller contact") from None + + def containers(self): + ids = self.docker.call("container", "ls", "--all", "--quiet", "--no-trunc").splitlines() + values = [] + for container_id in ids: + info = self.docker.inspect(container_id) + if info: + values.append(info) + return values + + def owned_container(self, info): + labels = info.get("Config", {}).get("Labels", {}) or {} + if labels.get(LABEL + ".managed") != "true": + return False + if labels.get(LABEL + ".host") not in (None, self.job["hostId"]): + return False + try: + instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", ""))) + except (AgentError, ValueError, TypeError): + return False + expected = str(self.root / "instances" / instance_id / str(generation) / "app") + # Legacy agents did not label the host; its exact managed bind mount + # proves ownership without touching another installation's containers. + return any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", [])) + + def uses_root(self, info): + for mount in info.get("Mounts", []): + if mount.get("Type") != "bind": + continue + source = Path(mount.get("Source", "/")).resolve() + if source == self.root or self.root in source.parents: + return True + return False + + def remove_docker_resources(self): + containers = self.containers() + if any(self.uses_root(info) and not self.owned_container(info) for info in containers): + raise AgentError("An unrelated container uses the host data directory; detach it before retrying deletion") + owned = [info for info in containers if self.owned_container(info)] + if any(info.get("State", {}).get("Running") is not False or info.get("State", {}).get("Status") not in ("created", "exited", "dead") for info in owned): + raise AgentError("A managed game container is still running or its state is unknown; retire it before deleting this host") + for info in owned: + # Do not force removal: Docker must also reject a container that + # starts again after inspection rather than interrupting players. + self.docker.call("container", "rm", "--volumes", info["Id"], timeout=120, tolerate_missing=True) + remaining = self.containers() + if any(self.uses_root(info) for info in remaining): + raise AgentError("Containers still use the host data directory") + # Never prune Docker globally. Only remove this agent's recorded image + # references, and preserve images used by any unrelated container. + used_images = {info.get("Image") for info in remaining} + images = self.job.get("runtimeImages", {}) + if not isinstance(images, dict): + raise AgentError("Managed runtime image ownership is unavailable") + for reference, expected_id in images.items(): + if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,511}", reference) or not isinstance(expected_id, str): + raise AgentError("Managed runtime image ownership is invalid") + raw = self.docker.call("image", "inspect", reference, tolerate_missing=True) + if raw is None: + continue + found = json.loads(raw) + if not found or found[0].get("Id") != expected_id or expected_id in used_images: + continue + self.docker.call("image", "rm", reference, tolerate_missing=True) + + def remove_data(self): + protected_path(AGENT_CONFIG.parent, missing=True) + if not self.job.get("agentStopped"): + systemctl("disable", "--now", AGENT_UNIT.name) + self.save(agentStopped=True) + self.remove_docker_resources() + root = uninstall_root(str(self.root), self.job["rootIdentity"]) + # This checkpoint also authorizes safe continuation if a power loss + # removes the ownership marker before the rest of the tree. + self.save(removalStarted=True) + if root.exists(): + if not shutil.rmtree.avoids_symlink_attacks: + raise AgentError("This platform cannot safely remove managed host files") + shutil.rmtree(root) + sync_parent(root) + for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT): + protected_file(path, missing=True) + path.unlink(missing_ok=True) + sync_parent(path) + systemctl("daemon-reload") + self.save(dataRemoved=True) + + def remove_helper(self): + # Disable does not stop this process: it must outlive its own unit and + # executable to deliver the final receipt after the files are gone. + protected_file(UNINSTALL_UNIT, missing=True) + if UNINSTALL_UNIT.exists(): + systemctl("disable", UNINSTALL_UNIT.name) + UNINSTALL_UNIT.unlink() + sync_parent(UNINSTALL_UNIT) + systemctl("daemon-reload") + protected_file(UNINSTALL_EXECUTABLE, missing=True) + UNINSTALL_EXECUTABLE.unlink(missing_ok=True) + sync_parent(UNINSTALL_EXECUTABLE) + protected_path(UNINSTALL_REQUEST.parent, missing=True) + if UNINSTALL_REQUEST.parent.exists(): + shutil.rmtree(UNINSTALL_REQUEST.parent) + sync_parent(UNINSTALL_REQUEST.parent) + + def run(self): + if not self.job.get("dataRemoved"): + try: + self.remove_data() + except (AgentError, OSError, ValueError, KeyError) as error: + with contextlib.suppress(AgentError): + self.receipt("failed", str(error) if isinstance(error, AgentError) else "Host cleanup could not finish. Check the uninstall service and resolve filesystem errors; cleanup will retry.") + raise + self.receipt("cleaned") + while True: + try: + self.remove_helper() + break + except (AgentError, OSError, ValueError): + # Once its unit/executable has been unlinked systemd can no + # longer restart this worker. Finish idempotently in memory. + with contextlib.suppress(AgentError): + self.receipt("failed", "Host files were removed, but removing the uninstall service is still retrying") + time.sleep(15) + # The controller never marks a host deleted on the earlier receipt. + # No credentials or retry state are left on disk at this point. + while True: + try: + self.receipt("completed") + return + except AgentError: + time.sleep(15) + + class Agent: def __init__(self, config, docker=None): self.config = config @@ -300,6 +580,43 @@ class Agent: self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json") self.executing_id = None + def can_uninstall(self): + try: + if Path(self.config_path) != AGENT_CONFIG: + return False + uninstall_ownership(self.config) + protected_file(UNINSTALL_EXECUTABLE) + protected_file(UNINSTALL_UNIT) + return True + except (AgentError, OSError, ValueError): + return False + + def decommission(self, operation): + payload = operation.get("payload", {}) + if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]: + raise Rejected("Host removal identity does not match this agent") + if payload.get("receiptUrl") != "/api/agent/decommission/" + identifier(operation["id"]) or not isinstance(payload.get("receiptToken"), str) or not payload["receiptToken"]: + raise Rejected("Invalid host removal receipt") + if not self.can_uninstall(): + raise Rejected("Reinstall the agent to enable safe removal of its dedicated host files") + root = uninstall_ownership(self.config) + info = root.stat() + protected_file(UNINSTALL_REQUEST, missing=True) + aliases_path = root / "agent" / "runtime-images.json" + protected_file(aliases_path, missing=True) + aliases = json.loads(aliases_path.read_text()) if aliases_path.exists() else {} + job = {"hostId": payload["hostId"], "operationId": operation["id"], "receiptUrl": payload["receiptUrl"], "receiptToken": payload["receiptToken"], "controller": self.config["controller"].rstrip("/"), "root": str(root), "rootIdentity": [info.st_dev, info.st_ino], "runtimeImages": aliases} + if UNINSTALL_REQUEST.exists(): + previous = json.loads(UNINSTALL_REQUEST.read_text()) + if any(previous.get(key) != job[key] for key in ("hostId", "operationId", "receiptUrl", "receiptToken", "controller", "root", "rootIdentity")): + raise Rejected("A different host removal is already pending") + self.checkpoint({"phase": "uninstall-service"}) + if not UNINSTALL_REQUEST.exists(): + atomic_json(UNINSTALL_REQUEST, job) + systemctl("enable", "--now", "--no-block", UNINSTALL_UNIT.name) + self.stopping.set() + return {"decommissioning": True, "message": "The uninstall service will report completion after host files and the agent are removed"} + def instance_dir(self, instance_id, generation): instance_id, generation = instance_identity(instance_id, generation) return self.root / "instances" / instance_id / str(generation) @@ -416,7 +733,7 @@ class Agent: for line in Path("/etc/os-release").read_text().splitlines(): if line.startswith("PRETTY_NAME="): os_name = line.split("=", 1)[1].strip('"') - return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} + return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} def safe_to_disrupt(self, instance_id, generation): health = self.inspect(instance_id, generation) @@ -525,6 +842,8 @@ class Agent: name = self.container_name(instance_id, generation) spec_sha = digest(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()) args = ["container", "create", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)] + if self.config.get("hostId"): + args.extend(["--label", LABEL + ".host=" + identifier(self.config["hostId"])]) bind_arg = "[" + bind + "]" if ":" in bind else bind for protocol in ("udp", "tcp"): args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"]) @@ -923,6 +1242,8 @@ class Agent: kind, payload = operation.get("kind"), operation.get("payload", {}) if not isinstance(kind, str) or not isinstance(payload, dict): raise AgentError("Invalid operation") + if kind == "host.decommission": + return self.decommission(operation) if kind == "credentials.rotate": pending = "ec_host_" + base64.urlsafe_b64encode(os.urandom(32)).rstrip(b"=").decode() self.config["pendingCredential"] = pending @@ -997,7 +1318,7 @@ class Agent: return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message) finally: self.executing_id = None - return self.journal.update(operation["id"], "completed", result) + return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result) def recover_credential(self): pending = self.config.get("pendingCredential") @@ -1024,6 +1345,13 @@ class Agent: instance_id, generation = operation.get("instanceId"), operation.get("generation") checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {} try: + if kind == "host.decommission": + self.executing_id = operation["id"] + try: + result = self.decommission(operation) + return self.journal.update(operation["id"], "running", result) + finally: + self.executing_id = None if kind == "credentials.rotate": if not self.config.get("pendingCredential") or self.recover_credential(): return self.journal.update(operation["id"], "completed", {"rotated": True, "reconciled": True}) @@ -1183,8 +1511,14 @@ def main(): parser.add_argument("--token-file") parser.add_argument("--root", default="/var/lib/electric-cherry") parser.add_argument("--force-enroll", action="store_true") + parser.add_argument("--uninstall", metavar="REQUEST", help="Run the installed host removal service") parser.add_argument("--version", action="version", version=VERSION) args = parser.parse_args() + if args.uninstall: + if os.geteuid() != 0 or args.enroll: + raise AgentError("Host uninstall must run as the dedicated root service") + Uninstaller(args.uninstall).run() + return if args.enroll: if not args.controller or not args.token_file: parser.error("Enrollment needs --controller and --token-file") diff --git a/agent/install.sh b/agent/install.sh index d96601d..8b249d1 100644 --- a/agent/install.sh +++ b/agent/install.sh @@ -21,7 +21,7 @@ done case "$controller" in https://*) ;; *) echo '--controller must be an HTTPS URL.' >&2; exit 2 ;; esac [ -n "$enrollment_token" ] || { echo '--token is required.' >&2; exit 2; } case "$data_dir" in /*) ;; *) echo '--data-dir must be absolute.' >&2; exit 2 ;; esac -case "$data_dir" in *[!a-zA-Z0-9/_.-]*|/|/etc|/usr|/var|/var/lib) echo 'Choose a dedicated data directory.' >&2; exit 2 ;; esac +case "$data_dir" in *[!a-zA-Z0-9/_.-]*|*/../*|*/./*|*/..|*/.|*//*|*/) echo 'Choose a canonical, dedicated data directory.' >&2; exit 2 ;; esac [ ! -L "$data_dir" ] || { echo 'Data directory must not be a symlink.' >&2; exit 2; } . /etc/os-release [ "${ID:-}" = debian ] || { echo 'Supported platform: Debian 12 or 13.' >&2; exit 1; } @@ -32,13 +32,82 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; } export DEBIAN_FRONTEND=noninteractive apt-get update -qq apt-get install -y --no-install-recommends ca-certificates curl python3 -python3 - "$controller" "$data_dir" <<'PY' -import json, pathlib, sys -config = pathlib.Path('/etc/electric-cherry/agent.json') -if config.exists(): - saved = json.loads(config.read_text()) - if saved.get('controller', '').rstrip('/') != sys.argv[1].rstrip('/') or pathlib.Path(saved.get('root', '')).resolve() != pathlib.Path(sys.argv[2]).resolve(): +python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY' +# INSTALLATION_PREFLIGHT: this runs before touching managed directories or services. +import json, pathlib, re, stat, sys + +controller, raw_root, config_name = sys.argv[1:] +root = pathlib.Path(raw_root) +config = pathlib.Path(config_name) +reserved = { + '/', '/bin', '/boot', '/dev', '/etc', '/home', '/lib', '/lib64', + '/media', '/mnt', '/opt', '/proc', '/root', '/run', '/sbin', '/srv', + '/sys', '/tmp', '/usr', '/usr/local', '/usr/local/bin', '/usr/local/lib', + '/usr/local/libexec', '/var', '/var/cache', '/var/lib', '/var/log', '/var/tmp', +} +if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root) + or str(root) != raw_root or any(part in ('.', '..') for part in raw_root.split('/')[1:]) + or len(root.parts) < 3 or raw_root in reserved or config.parent == root or config.parent in root.parents): + raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.') +installed_paths = [config, + pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'), + pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'), + pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'), + pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service')] +if any(root == path or root in path.parents for path in installed_paths): + raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.') + +def trusted_directories(path): + for current in [*reversed(path.parents), path]: + try: + info = current.lstat() + except FileNotFoundError: + continue + if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022: + raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.') + +def trusted_json(path): + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022: + raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.') + value = json.loads(path.read_text()) + if not isinstance(value, dict): + raise SystemExit('Existing installation metadata is invalid.') + return value + +trusted_directories(root) +trusted_directories(config.parent) +mountinfo = pathlib.Path('/proc/1/mountinfo') +if mountinfo.exists(): + for line in mountinfo.read_text().splitlines(): + fields = line.split() + if len(fields) < 5: + continue + mount = re.sub(r'\\([0-7]{3})', lambda match: chr(int(match[1], 8)), fields[4]) + if mount == raw_root or mount.startswith(raw_root + '/'): + raise SystemExit('Use a dedicated subdirectory of the mounted disk, with no filesystems mounted inside it, so host deletion can remove the entire data directory.') +request = config.with_name('decommission.json') +if request.exists() or request.is_symlink(): + raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.') +saved = None +if config.exists() or config.is_symlink(): + saved = trusted_json(config) + if saved.get('controller', '').rstrip('/') != controller.rstrip('/') or saved.get('root') != raw_root: raise SystemExit('Existing enrollment uses a different controller or data directory. Preserve the existing installation and migrate it explicitly.') + if not saved.get('hostId') or not saved.get('credential'): + raise SystemExit('Existing enrollment is incomplete. Repair its credentials before upgrading.') +marker = root / '.electric-cherry.json' +if marker.exists() or marker.is_symlink(): + owned = trusted_json(marker) + if not saved or owned != {'version': 1, 'root': raw_root, 'hostId': saved['hostId'], 'controller': controller.rstrip('/')}: + raise SystemExit('Data directory ownership does not match this enrollment. Preserve it and migrate explicitly.') +elif root.exists() and any(root.iterdir()): + # Upgrade installations predating ownership markers only when the entire + # root has the known layout. Never adopt a directory of unrelated files. + if not saved or any(path.name not in {'agent', 'instances', 'libraries'} for path in root.iterdir()): + raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.') +for directory in ('agent', 'instances', 'libraries'): + trusted_directories(root / directory) PY if ! command -v docker >/dev/null; then if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then @@ -76,9 +145,36 @@ install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry- install -d -m 0755 /usr/local/libexec install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py } +# A separate worker survives stopping/removing the agent service. Its only +# entry point is a fixed, authenticated deletion request prepared by the agent. +install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token" unset enrollment_token python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"} +python3 - "$data_dir" /etc/electric-cherry/agent.json <<'PY' +# INSTALLATION_OWNERSHIP: bind deletion authority to this exact enrollment/root. +import json, os, pathlib, sys, uuid +root = pathlib.Path(sys.argv[1]) +config = json.loads(pathlib.Path(sys.argv[2]).read_text()) +if config.get('root') != str(root) or not config.get('hostId') or not config.get('controller'): + raise SystemExit('Enrollment did not establish ownership of this data directory.') +marker = root / '.electric-cherry.json' +temporary = root / ('.electric-cherry.' + uuid.uuid4().hex + '.tmp') +try: + with temporary.open('x') as stream: + os.chmod(temporary, 0o600) + json.dump({'version': 1, 'root': str(root), 'hostId': config['hostId'], 'controller': config['controller'].rstrip('/')}, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, marker) + directory_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) +finally: + temporary.unlink(missing_ok=True) +PY cat > /etc/systemd/system/electric-cherry-agent.service < /etc/systemd/system/electric-cherry-uninstall.service <<'EOF' +[Unit] +Description=Remove an Electric Cherry Host Installation +After=network-online.target docker.service +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-uninstall.py --uninstall /etc/electric-cherry/decommission.json +Restart=on-failure +RestartSec=15 +TimeoutStartSec=0 +UMask=0077 + [Install] WantedBy=multi-user.target EOF diff --git a/agent/package_release.py b/agent/package_release.py index 64ec0af..d06464b 100644 --- a/agent/package_release.py +++ b/agent/package_release.py @@ -10,4 +10,4 @@ target = source.parent / "static" / "agent" target.mkdir(parents=True, exist_ok=True) shutil.copyfile(source / "ec_agent.py", target / "ec-agent.py") shutil.copyfile(source / "install.sh", target / "install.sh") -(target / "manifest.json").write_text(json.dumps({"version": "1.0.0", "protocolVersion": 1, "platform": "debian-amd64", "sha256": hashlib.sha256((target / "ec-agent.py").read_bytes()).hexdigest()}, indent=2) + "\n") +(target / "manifest.json").write_text(json.dumps({"version": "1.1.0", "protocolVersion": 1, "platform": "debian-amd64", "sha256": hashlib.sha256((target / "ec-agent.py").read_bytes()).hexdigest()}, indent=2) + "\n") diff --git a/agent/tests/test_installer.py b/agent/tests/test_installer.py new file mode 100644 index 0000000..807dddd --- /dev/null +++ b/agent/tests/test_installer.py @@ -0,0 +1,185 @@ +"""Exercise installer guards without installing software or touching services.""" + +import json +import os +from pathlib import Path +import re +import stat +import sys +import tempfile +import unittest +from unittest.mock import patch + + +SOURCE = (Path(__file__).resolve().parents[1] / "install.sh").read_text() +BLOCKS = re.findall(r"<<'PY'\n(.*?)\nPY", SOURCE, re.DOTALL) +PREFLIGHT = next(block for block in BLOCKS if "# INSTALLATION_PREFLIGHT:" in block) +OWNERSHIP = next(block for block in BLOCKS if "# INSTALLATION_OWNERSHIP:" in block) + + +class InstallerTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.base = Path(self.temporary.name).resolve() + self.root = self.base / "managed" + self.config = self.base / "configuration" / "agent.json" + self.config.parent.mkdir() + self.controller = "https://controller.example" + self.saved = {"root": str(self.root), "controller": self.controller, + "hostId": "host-one", "credential": "private"} + self.overrides = {} + + def tearDown(self): + self.temporary.cleanup() + + def preflight(self, root=None): + original_lstat = Path.lstat + + def root_owned(path): + # Simulate Linux root-owned parents while keeping the actual inode + # types. No filesystem mutations or elevated privileges are needed. + info = original_lstat(path) + fields = list(info) + fields[4] = 0 + if stat.S_ISDIR(info.st_mode): + fields[0] &= ~0o022 + override = self.overrides.get(str(path), {}) + fields[0] = override.get("mode", fields[0]) + fields[4] = override.get("uid", fields[4]) + return os.stat_result(fields) + + argv = ["-", self.controller, str(root or self.root), str(self.config)] + with patch.object(sys, "argv", argv), patch.object(Path, "lstat", root_owned): + exec(compile(PREFLIGHT, "installer-preflight", "exec"), {}) + + def legacy_installation(self): + for directory in ("agent", "instances", "libraries"): + (self.root / directory).mkdir(parents=True, exist_ok=True) + self.config.write_text(json.dumps(self.saved)) + + def ownership(self): + with patch.object(sys, "argv", ["-", str(self.root), str(self.config)]): + exec(compile(OWNERSHIP, "installer-ownership", "exec"), {}) + + def test_new_directory_and_empty_directory_are_allowed_without_adopting_files(self): + self.preflight() + self.assertFalse(self.root.exists()) + self.root.mkdir() + self.preflight() + (self.root / "unrelated.txt").write_text("keep") + with self.assertRaisesRegex(SystemExit, "not owned"): + self.preflight() + self.assertEqual((self.root / "unrelated.txt").read_text(), "keep") + + def test_known_legacy_layout_can_upgrade_and_marker_binds_exact_enrollment(self): + self.legacy_installation() + original_config = self.config.read_bytes() + (self.root / "libraries" / "game.ff").write_bytes(b"game data") + self.preflight() + self.ownership() + marker = self.root / ".electric-cherry.json" + self.assertEqual(json.loads(marker.read_text()), { + "version": 1, "root": str(self.root), "hostId": "host-one", "controller": self.controller}) + self.assertEqual(stat.S_IMODE(marker.stat().st_mode), 0o600) + self.assertEqual(self.config.read_bytes(), original_config) + self.assertEqual((self.root / "libraries" / "game.ff").read_bytes(), b"game data") + self.preflight() + + def test_legacy_upgrade_never_adopts_unrelated_top_level_files(self): + self.legacy_installation() + (self.root / "personal-backup.tar").write_bytes(b"keep") + with self.assertRaisesRegex(SystemExit, "not owned"): + self.preflight() + self.assertFalse((self.root / ".electric-cherry.json").exists()) + + def test_reenrollment_cannot_silently_change_controller_or_root(self): + self.legacy_installation() + for field, value in (("controller", "https://another.example"), ("root", str(self.base / "other"))): + with self.subTest(field=field): + self.config.write_text(json.dumps({**self.saved, field: value})) + with self.assertRaisesRegex(SystemExit, "different controller or data directory"): + self.preflight() + + def test_existing_marker_must_match_controller_host_and_root(self): + self.legacy_installation() + self.ownership() + marker = self.root / ".electric-cherry.json" + valid = json.loads(marker.read_text()) + for field, value in (("hostId", "host-two"), ("root", str(self.base)), ("controller", "https://another.example")): + with self.subTest(field=field): + marker.write_text(json.dumps({**valid, field: value})) + with self.assertRaisesRegex(SystemExit, "ownership does not match"): + self.preflight() + + def test_lexical_aliases_and_system_roots_are_rejected_before_mutation(self): + for root in ("/var/lib/..", "/var/lib/../lib", "/var//lib/managed", "/var/lib/./managed", "/var/lib/managed/", "/var/lib", "/usr/local/libexec", "/", "/games", str(self.config.parent), str(self.config.parent / "data")): + with self.subTest(root=root), self.assertRaisesRegex(SystemExit, "canonical, dedicated"): + self.preflight(root) + self.assertFalse(self.root.exists()) + + def test_symlink_roots_ancestors_metadata_and_managed_subdirectories_are_rejected(self): + elsewhere = self.base / "elsewhere" + elsewhere.mkdir() + self.root.symlink_to(elsewhere, target_is_directory=True) + with self.assertRaisesRegex(SystemExit, "not symlinks"): + self.preflight() + with self.assertRaisesRegex(SystemExit, "not symlinks"): + self.preflight(str(self.root / "nested")) + self.root.unlink() + self.legacy_installation() + (self.root / "libraries").rmdir() + (self.root / "libraries").symlink_to(elsewhere, target_is_directory=True) + with self.assertRaisesRegex(SystemExit, "not symlinks"): + self.preflight() + (self.root / "libraries").unlink() + (self.root / "libraries").mkdir() + self.config.unlink() + real_config = self.base / "real-config.json" + real_config.write_text(json.dumps(self.saved)) + self.config.symlink_to(real_config) + with self.assertRaisesRegex(SystemExit, "regular, root-owned"): + self.preflight() + + def test_untrusted_parents_and_metadata_cannot_authorize_cleanup(self): + self.legacy_installation() + self.overrides[str(self.base)] = {"uid": 1000} + with self.assertRaisesRegex(SystemExit, "root-owned"): + self.preflight() + self.overrides = {str(self.base): {"mode": stat.S_IFDIR | 0o777}} + with self.assertRaisesRegex(SystemExit, "root-owned"): + self.preflight() + self.overrides = {str(self.config): {"uid": 1000}} + with self.assertRaisesRegex(SystemExit, "regular, root-owned"): + self.preflight() + + def test_pending_deletion_blocks_upgrade_even_with_dangling_request_symlink(self): + self.legacy_installation() + request = self.config.with_name("decommission.json") + request.write_text("{}") + with self.assertRaisesRegex(SystemExit, "deletion is in progress"): + self.preflight() + request.unlink() + request.symlink_to(self.base / "missing") + with self.assertRaisesRegex(SystemExit, "deletion is in progress"): + self.preflight() + + def test_mounted_parent_is_allowed_but_data_root_and_nested_mounts_are_rejected(self): + mountinfo = Path('/proc/1/mountinfo') + original_exists, original_read = Path.exists, Path.read_text + for mount, allowed in ((self.base, True), (self.root, False), (self.root / 'libraries', False)): + def exists(path): + return True if path == mountinfo else original_exists(path) + + def read(path, *args, **kwargs): + return f'29 24 8:1 / {mount} rw - ext4 /dev/sda1 rw' if path == mountinfo else original_read(path, *args, **kwargs) + + with self.subTest(mount=mount), patch.object(Path, 'exists', exists), patch.object(Path, 'read_text', read): + if allowed: + self.preflight() + else: + with self.assertRaisesRegex(SystemExit, "dedicated subdirectory of the mounted disk"): + self.preflight() + + +if __name__ == "__main__": + unittest.main() diff --git a/agent/tests/test_uninstall.py b/agent/tests/test_uninstall.py new file mode 100644 index 0000000..aa1507e --- /dev/null +++ b/agent/tests/test_uninstall.py @@ -0,0 +1,338 @@ +import importlib.util +import json +import os +from pathlib import Path +import shutil +import stat +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("ec_uninstall_agent", Path(__file__).resolve().parents[1] / "ec_agent.py") +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + +INSTANCE = "a53d99da-a1eb-4f01-9b10-a6268b556542" + + +class CleanupDocker: + def __init__(self): + self.calls = [] + self.items = {} + self.images = {} + + def inspect(self, identity): + return self.items.get(identity) + + def call(self, *args, **kwargs): + self.calls.append(args) + if args[:2] == ("container", "ls"): + return "\n".join(self.items) + if args[:2] == ("container", "rm"): + self.items.pop(args[-1], None) + return "" + if args[:2] == ("image", "inspect"): + item = self.images.get(args[-1]) + return json.dumps([item]) if item else None + if args[:2] == ("image", "rm"): + self.images.pop(args[-1], None) + return "" + raise AssertionError("Unexpected Docker request: " + str(args)) + + +class UninstallTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.base = Path(self.temp.name).resolve() + self.root = self.base / "host-data" + self.config_dir = self.base / "etc" / "electric-cherry" + self.bin_dir = self.base / "libexec" + self.unit_dir = self.base / "systemd" + for directory in (self.config_dir, self.bin_dir, self.unit_dir): + directory.mkdir(parents=True) + replacements = { + "AGENT_CONFIG": self.config_dir / "agent.json", + "AGENT_EXECUTABLE": self.bin_dir / "electric-cherry-agent.py", + "AGENT_UNIT": self.unit_dir / "electric-cherry-agent.service", + "UNINSTALL_EXECUTABLE": self.bin_dir / "electric-cherry-uninstall.py", + "UNINSTALL_UNIT": self.unit_dir / "electric-cherry-uninstall.service", + "UNINSTALL_REQUEST": self.config_dir / "decommission.json", + } + self.patches = [patch.object(module, key, value) for key, value in replacements.items()] + # Unit tests run without root privileges. Keep actual path, symlink, + # file-type and containment checks while substituting owner checks. + def test_protected_path(value, missing=False): + path = Path(value) + if any(current.is_symlink() for current in (path, *path.parents)): + raise module.AgentError("Symlink ownership is invalid") + if not missing and not path.exists(): + raise module.AgentError("Host uninstall files are unavailable") + return path + self.patches.extend([patch.object(module, "protected_path", side_effect=test_protected_path), patch.object(module, "systemctl"), patch.object(module, "sync_parent")]) + for entry in self.patches: + entry.start() + self.config = {"hostId": "host-one", "controller": "https://controller.example", "credential": "secret", "root": str(self.root), "configPath": str(module.AGENT_CONFIG)} + self.agent = module.Agent(self.config) + self.docker = CleanupDocker() + for path in (module.AGENT_EXECUTABLE, module.AGENT_UNIT, module.UNINSTALL_EXECUTABLE, module.UNINSTALL_UNIT): + path.write_text("installed") + module.atomic_json(module.AGENT_CONFIG, self.config) + module.atomic_json(self.root / module.ROOT_MARKER, {"version": 1, "root": str(self.root), "hostId": "host-one", "controller": "https://controller.example"}) + self.game = self.root / "libraries" / "t6" / "library" / "content" / "zone" / "all" / "dlc.ff" + self.game.parent.mkdir(parents=True) + self.game.write_bytes(b"game DLC") + (self.base / "unrelated.txt").write_text("keep") + self.operation = {"id": "delete-one", "kind": "host.decommission", "payload": {"hostId": "host-one", "operationId": "delete-one", "receiptToken": "receipt-only-secret", "receiptUrl": "/api/agent/decommission/delete-one"}} + + def tearDown(self): + self.agent.journal.close() + for entry in reversed(self.patches): + entry.stop() + self.temp.cleanup() + + def prepare(self): + result = self.agent.execute(self.operation) + self.assertEqual(result["state"], "running", result) + return module.Uninstaller(module.UNINSTALL_REQUEST, self.docker) + + def container(self, identity, owned=True, host="host-one", root=None, image="image-one"): + labels = {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1", module.LABEL + ".host": host} if owned else {} + return {"Id": identity, "Image": image, "Config": {"Labels": labels}, "State": {"Running": False, "Status": "exited"}, "Mounts": [{"Type": "bind", "Source": str((root or self.root) / "instances" / INSTANCE / "1" / "app"), "Destination": "/home/plutainer/app"}]} + + def test_final_receipt_follows_data_and_self_removal(self): + worker = self.prepare() + stages = [] + def receipt(stage, error=None): + stages.append(stage) + self.assertFalse(self.root.exists()) + self.assertFalse(module.AGENT_CONFIG.exists()) + self.assertFalse(module.AGENT_EXECUTABLE.exists()) + self.assertFalse(module.AGENT_UNIT.exists()) + if stage == "cleaned": + self.assertTrue(module.UNINSTALL_REQUEST.exists()) + self.assertTrue(module.UNINSTALL_EXECUTABLE.exists()) + else: + self.assertFalse(self.config_dir.exists()) + self.assertFalse(module.UNINSTALL_EXECUTABLE.exists()) + self.assertFalse(module.UNINSTALL_UNIT.exists()) + worker.receipt = receipt + worker.run() + self.assertEqual(stages, ["cleaned", "completed"]) + self.assertEqual((self.base / "unrelated.txt").read_text(), "keep") + module.systemctl.assert_any_call("disable", "--now", "electric-cherry-agent.service") + module.systemctl.assert_any_call("disable", "electric-cherry-uninstall.service") + + def test_handoff_is_durable_and_never_reports_operation_completed(self): + worker = self.prepare() + self.assertEqual(worker.job["operationId"], "delete-one") + self.assertNotIn("credential", worker.job) + self.assertTrue(self.agent.stopping.is_set()) + module.systemctl.assert_any_call("enable", "--now", "--no-block", "electric-cherry-uninstall.service") + self.assertEqual(self.agent.journal.get("delete-one")["state"], "running") + + def test_failed_checkpoint_cannot_leave_a_runnable_uninstall_request(self): + with patch.object(self.agent.journal, "checkpoint", side_effect=OSError("journal unavailable")): + result = self.agent.execute(self.operation) + self.assertEqual(result["state"], "failed") + self.assertFalse(module.UNINSTALL_REQUEST.exists()) + module.systemctl.assert_not_called() + + def test_pending_cleanup_resumes_after_controller_unavailable(self): + worker = self.prepare() + worker.receipt = lambda *_: (_ for _ in ()).throw(module.AgentError("offline")) + with self.assertRaisesRegex(module.AgentError, "offline"): + worker.run() + self.assertFalse(self.root.exists()) + self.assertTrue(module.UNINSTALL_REQUEST.exists()) + self.assertTrue(module.UNINSTALL_EXECUTABLE.exists()) + self.assertTrue(json.loads(module.UNINSTALL_REQUEST.read_text())["dataRemoved"]) + before = len(self.docker.calls) + resumed = module.Uninstaller(module.UNINSTALL_REQUEST, self.docker) + stages = [] + resumed.receipt = lambda stage, error=None: stages.append(stage) + resumed.run() + self.assertEqual(stages, ["cleaned", "completed"]) + self.assertEqual(len(self.docker.calls), before) + + def test_lost_final_receipt_retries_after_all_files_are_gone(self): + worker = self.prepare() + stages = [] + def receipt(stage, error=None): + stages.append(stage) + if stage == "completed": + self.assertFalse(self.config_dir.exists()) + if stages.count("completed") == 1: + raise module.AgentError("lost reply") + worker.receipt = receipt + with patch.object(module.time, "sleep") as wait: + worker.run() + self.assertEqual(stages, ["cleaned", "completed", "completed"]) + wait.assert_called_once_with(15) + + def test_partial_self_removal_retries_without_its_unit_or_executable(self): + worker = self.prepare() + stages = [] + worker.receipt = lambda stage, error=None: stages.append(stage) + original = shutil.rmtree + attempts = 0 + def interrupted(path, *args, **kwargs): + nonlocal attempts + if Path(path) == self.config_dir: + attempts += 1 + if attempts == 1: + self.assertFalse(module.UNINSTALL_UNIT.exists()) + self.assertFalse(module.UNINSTALL_EXECUTABLE.exists()) + raise OSError("temporarily busy") + return original(path, *args, **kwargs) + with patch.object(module.shutil, "rmtree", side_effect=interrupted), patch.object(module.time, "sleep") as wait: + worker.run() + self.assertEqual(stages, ["cleaned", "failed", "completed"]) + self.assertFalse(self.config_dir.exists()) + wait.assert_called_once_with(15) + + def test_partial_tree_removal_recovers_without_deleted_marker(self): + worker = self.prepare() + original = shutil.rmtree + def interrupted(path, *args, **kwargs): + if Path(path) == self.root: + (self.root / module.ROOT_MARKER).unlink() + self.game.unlink() + raise OSError("power loss") + return original(path, *args, **kwargs) + worker.receipt = lambda *_: None + with patch.object(module.shutil, "rmtree", side_effect=interrupted): + with self.assertRaisesRegex(OSError, "power loss"): + worker.run() + self.assertTrue(json.loads(module.UNINSTALL_REQUEST.read_text())["removalStarted"]) + self.assertFalse((self.root / module.ROOT_MARKER).exists()) + resumed = module.Uninstaller(module.UNINSTALL_REQUEST, self.docker) + resumed.receipt = lambda *_: None + resumed.run() + self.assertFalse(self.root.exists()) + + def test_replaced_data_root_cannot_be_deleted_on_resume(self): + self.prepare() + self.root.rename(self.base / "original-data") + self.root.mkdir() + (self.root / "precious.txt").write_text("keep") + with self.assertRaisesRegex(module.AgentError, "changed"): + module.Uninstaller(module.UNINSTALL_REQUEST, self.docker) + self.assertTrue((self.root / "precious.txt").exists()) + + def test_only_owned_containers_and_unused_recorded_images_are_removed(self): + aliases = {"example/runtime:one": "image-one", "example/runtime:shared": "image-shared", "example/runtime:changed": "image-old"} + module.atomic_json(self.root / "agent" / "runtime-images.json", aliases) + self.docker.images = {"example/runtime:one": {"Id": "image-one"}, "example/runtime:shared": {"Id": "image-shared"}, "example/runtime:changed": {"Id": "image-new"}, "example/unrelated:one": {"Id": "image-unrelated"}} + self.docker.items = {"ours": self.container("ours"), "other": self.container("other", host="host-two", root=self.base / "different-install", image="image-shared")} + worker = self.prepare() + worker.receipt = lambda *_: None + worker.run() + self.assertNotIn("ours", self.docker.items) + self.assertIn("other", self.docker.items) + self.assertNotIn("example/runtime:one", self.docker.images) + self.assertIn("example/runtime:shared", self.docker.images) + self.assertIn("example/runtime:changed", self.docker.images) + self.assertIn("example/unrelated:one", self.docker.images) + self.assertFalse(any("prune" in call or "volume" in call for call in self.docker.calls)) + + def test_unrelated_container_using_root_blocks_any_removal(self): + self.docker.items = {"foreign": self.container("foreign", owned=False)} + worker = self.prepare() + stages = [] + worker.receipt = lambda stage, error=None: stages.append((stage, error)) + with self.assertRaisesRegex(module.AgentError, "unrelated container"): + worker.run() + self.assertEqual(stages[0][0], "failed") + self.assertTrue(self.game.exists()) + self.assertTrue(module.AGENT_EXECUTABLE.exists()) + self.assertIn("foreign", self.docker.items) + + def test_running_or_unknown_managed_container_blocks_removal(self): + worker = self.prepare() + worker.receipt = lambda *_: None + for state in ({"Running": True, "Status": "running"}, {"Running": False, "Status": "restarting"}, {}): + with self.subTest(state=state): + self.docker.items = {"ours": {**self.container("ours"), "State": state}} + with self.assertRaisesRegex(module.AgentError, "still running or its state is unknown"): + worker.run() + self.assertTrue(self.game.exists()) + self.assertIn("ours", self.docker.items) + self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls)) + + def test_legacy_container_requires_exact_managed_bind_mount(self): + worker = self.prepare() + legacy = self.container("legacy") + legacy["Config"]["Labels"].pop(module.LABEL + ".host") + self.assertTrue(worker.owned_container(legacy)) + legacy["Mounts"][0]["Source"] = str(self.base / "elsewhere") + self.assertFalse(worker.owned_container(legacy)) + + def test_missing_marker_and_wrong_host_never_schedule_helper(self): + (self.root / module.ROOT_MARKER).unlink() + self.assertFalse(self.agent.can_uninstall()) + result = self.agent.execute(self.operation) + self.assertEqual(result["state"], "failed") + module.systemctl.assert_not_called() + self.assertTrue(self.game.exists()) + + def test_receipt_path_and_identity_cannot_be_redirected(self): + for key, value in (("receiptUrl", "https://attacker.example"), ("hostId", "other"), ("operationId", "other")): + with self.subTest(key=key): + operation = {**self.operation, "id": "check-" + key, "payload": {**self.operation["payload"], "operationId": "check-" + key, "receiptUrl": "/api/agent/decommission/check-" + key, key: value}} + self.assertEqual(self.agent.execute(operation)["state"], "failed") + module.systemctl.assert_not_called() + + def test_symlink_inside_owned_tree_is_unlinked_without_following(self): + outside = self.base / "unrelated.txt" + (self.root / "libraries" / "symlink").symlink_to(outside) + worker = self.prepare() + worker.receipt = lambda *_: None + worker.run() + self.assertEqual(outside.read_text(), "keep") + + def test_unsafe_roots_and_installation_overlap_are_rejected(self): + for value in ("/", "/var/lib", "/srv", "/var/lib/../lib", "/var//lib/anything", str(self.config_dir), str(self.bin_dir)): + with self.subTest(value=value): + with self.assertRaises(module.AgentError): + module.uninstall_root(value) + + def test_root_mount_or_nested_mount_cannot_be_recursively_deleted(self): + original = Path.read_text + for mount in (self.root, self.root / "libraries"): + def read_text(path, *args, **kwargs): + if str(path) == "/proc/1/mountinfo": + return "21 1 8:1 / " + str(mount) + " rw - ext4 /dev/sda1 rw\n" + return original(path, *args, **kwargs) + with patch.object(Path, "read_text", read_text): + with self.assertRaisesRegex(module.AgentError, "Unmount"): + module.uninstall_root(str(self.root)) + + def test_marker_directory_is_rejected_before_reading(self): + marker = self.root / module.ROOT_MARKER + marker.unlink() + marker.mkdir() + with self.assertRaisesRegex(module.AgentError, "regular file"): + module.uninstall_ownership(self.config) + + def test_boot_reconciles_handoff_instead_of_marking_complete(self): + self.prepare() + self.agent.journal.close() + self.agent = module.Agent(self.config) + self.assertEqual(self.agent.journal.get("delete-one")["state"], "unknown") + result = self.agent.execute(self.operation) + self.assertEqual(result["state"], "running") + self.assertEqual(json.loads(module.UNINSTALL_REQUEST.read_text())["operationId"], "delete-one") + + +class OwnershipPathTests(unittest.TestCase): + def test_root_ownership_group_writes_and_symlinks_are_rejected(self): + for mode, uid in ((stat.S_IFDIR | 0o777, 0), (stat.S_IFDIR | 0o755, 1000), (stat.S_IFLNK | 0o777, 0)): + with self.subTest(mode=mode, uid=uid): + value = os.stat_result((mode, 1, 1, 1, uid, 0, 0, 0, 0, 0)) + with patch.object(Path, "lstat", return_value=value): + with self.assertRaisesRegex(module.AgentError, "only by root"): + module.protected_path("/var/lib/electric-cherry") + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/authentication.md b/docs/authentication.md index 492739c..53be147 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -85,6 +85,9 @@ endpoint or accept passwords as command arguments. Host agents retain enrollment/host-token authentication under `/api/agent/`. Installer and release files remain public so a new machine can enroll. A browser login never substitutes for a host credential, and host tokens cannot manage EC. +Host deletion uses a separate operation-scoped receipt token so the uninstall +worker can confirm removal and retry its acknowledgment after host credentials +are revoked. That token authorizes only the corresponding cleanup receipt. Game reporters must send a separate secret of at least 32 characters configured as `EC_TELEMETRY_TOKEN`: diff --git a/docs/native-hosts.md b/docs/native-hosts.md index 5464d7b..cf227c7 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -35,7 +35,10 @@ A fresh destination host downloads centrally stored game files without manual copying or per-container setup. EC does not supply the game content itself. The default host root is `/var/lib/electric-cherry`; installer `--data-dir` changes -it. Data is separated into `libraries///content`, +it. Use a dedicated, root-owned directory. For a separate disk, use a child of its +mount point (for example `/mnt/games/electric-cherry`), with no nested mounts, so +host deletion can remove the entire directory. The installer records ownership +and refuses to adopt unrelated files. Data is separated into `libraries///content`, `instances///app`, and `agent/operations.sqlite3`. Base libraries are mounted read-only and instance deletion removes only that instance's files. The controller stores large files under `GAME_FILES_ROOT` (default beside the @@ -74,12 +77,15 @@ explicit installer `--reenroll` when local credentials already exist. API outline: - `GET /api/hosts/:id`: host details, operation history and deletion blockers. -- `DELETE /api/hosts/:id`: administrator removes an unused host and revokes access; - returns `409` if servers, reservations or unresolved delivered operations remain. +- `DELETE /api/hosts/:id`: administrator requests cleanup of an unused host; + returns `202` while remote removal is pending, `200` for an unenrolled host, + or `409` when the agent needs updating or unresolved ownership remains. - `POST /api/agent/enroll`: consume enrollment token and return host credentials. - `POST /api/agent/heartbeat`: authenticated inventory, current operation and caches. - `GET /api/agent/operations`: deliver/reconcile durable operation IDs. - `POST /api/agent/operations/:id`: authenticated acceptance, progress or result. +- `POST /api/agent/decommission/:id`: cleanup worker reports file removal and + agent uninstall using its own operation-scoped receipt credential. - `POST /api/game-files`: administrator creates a folder upload with game, name, version, optional description and `files: [{path, size, lastModified}]`. - `GET /api/game-files/:id/upload`: durable per-file offsets and publication status. @@ -133,13 +139,34 @@ Administrators can choose **Delete host** on a host card or at the top of its detail page. The confirmation lists anything that must be resolved first: registered servers, reserved ports, or operations already sent to the agent. Drain the host or remove its servers, then let outstanding operations reconcile. -An unused host can be deleted before enrollment or while offline. +An unenrolled host is removed immediately. An enrolled host stays visible as +**Deleting** until its agent confirms cleanup. Offline hosts wait for their next +connection; their absence is never treated as proof of removal. Existing agents +need the 1.1.0 installer: generate a fresh command under **Agent access**, rerun it +on the host, and wait for a heartbeat advertising uninstall support. -Deleting a host removes it from the controller's host list and deployment -choices, revokes its credentials, and invalidates unused installer links. -Queued commands that have never been sent to the agent are cancelled. Operation -history and published libraries are retained; the agent and game files remain on -the machine. Adding the machine again requires a new host enrollment. +Deletion blocks new operations, cancels queued commands that were never delivered, +and invalidates unused installer links. A separate cleanup service removes the +host's downloaded game libraries, instance files, agent journal and configuration, +owned containers, and recorded runtime images unused by other containers. It +stops and removes the agent service and executable, then removes its own files. +The controller hides the host and revokes its ordinary credentials only after +receiving the final uninstall receipt. Adding the machine again requires a new +host enrollment. + +Game folders uploaded through Electric Cherry belong to the controller. Those +original uploads remain available for other hosts and future deployments; only +the deleted host's copies are removed. Controller operation history is also kept. +Unrelated containers, files and shared dependencies such as Docker and Python +are preserved. Historical entries in shared system logs are not erased. + +Cleanup retries after connection loss or partial file removal. Failures before +handoff offer **Retry cleanup** after the installation is repaired; failures in +the independent worker retry automatically. The worker stays installed until +the controller acknowledges data cleanup. After removing its final files it +retries the completion receipt from memory. A power loss in that final window +can leave the controller awaiting confirmation even though removal succeeded; +inspect the machine if that status persists. The controller does not guess. ## Libraries and ports @@ -198,7 +225,9 @@ Automated tests cover enrollment/authentication, rotation/revocation, host readiness, scoped ports, immutable library publication, streaming/ranges, checksums, interrupted downloads, instance generation ownership, local occupancy holds, controller restart/reconnect, ambiguous results, occupied campaign staging, -activation before images, collections and draining. Docker/UDP and host +activation before images, collections and draining. Host deletion tests cover +ownership guards, interrupted cleanup, uninstall receipts, credential revocation +and retention of controller uploads/history. Docker/systemd/UDP and host observations are simulated in these tests. A real Debian/Docker/Plutonium acceptance run is still required before production @@ -213,6 +242,10 @@ coverage is not evidence that a real T5/T6 server has been deployed here. Run: agent activation, and during file activation before restart; verify that the agent holds disruption. - Drain occupied lobbies and verify shared libraries survive their retirement. +- Delete a drained host; verify its game copies, data directory, agent and cleanup + services/configuration/executables are removed, controller uploads/history stay, + and unrelated Docker workloads continue. Repeat with connection loss during + cleanup and after the final receipt, and a reboot during data removal. - Corrupt/interrupted library downloads, resumed transfer and verified promotion. - Controller restart, agent restart and network loss before execution, during execution and after execution but before acknowledgement. Verify the same diff --git a/src/lib/hosts.ts b/src/lib/hosts.ts index b99b9eb..2a25acb 100644 --- a/src/lib/hosts.ts +++ b/src/lib/hosts.ts @@ -1,13 +1,27 @@ export type HostMode = 'active' | 'draining' | 'disabled'; -export type HostReadiness = 'preparing' | 'needs-runtime' | 'needs-game-files' | 'ready' | 'degraded' | 'draining' | 'disabled'; +export type HostReadiness = 'preparing' | 'needs-runtime' | 'needs-game-files' | 'ready' | 'degraded' | 'draining' | 'disabled' | 'deleting'; export type HostOperationState = 'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown'; export function hostReadinessLabel(readiness: HostReadiness): string { return { preparing: 'Preparing', 'needs-runtime': 'Needs runtime', 'needs-game-files': 'Needs game files', - ready: 'Ready', degraded: 'Degraded', draining: 'Draining', disabled: 'Disabled' }[readiness]; + ready: 'Ready', degraded: 'Degraded', draining: 'Draining', disabled: 'Disabled', deleting: 'Deleting' }[readiness]; } export interface HostDeletion { allowed: boolean; blockers: string[] } +export interface HostDecommission { + operationId: string; stage: 'queued' | 'running' | 'cleaned' | 'failed' | 'completed'; + requestedAt: string; updatedAt: string; error: string | null; retryable: boolean; +} +export function hostDeletionMessage(host: Pick): string { + if (!host.decommission) return ''; + switch (host.decommission.stage) { + case 'queued': return host.connected ? 'Host cleanup is queued.' : 'Waiting for the host to reconnect before removing its files and agent.'; + case 'running': return 'Removing host files, cached game libraries, and the agent installation.'; + case 'cleaned': return 'Host data is removed. Waiting for confirmation that the agent has uninstalled itself.'; + case 'failed': return 'Cleanup needs attention. Complete removal has not been confirmed.'; + case 'completed': return 'Host files and agent removal confirmed.'; + } +} export interface HostLibrary { id: string; state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; progress?: number; error?: string } export interface HostInventory { @@ -16,6 +30,7 @@ export interface HostInventory { storageTotal?: number; storageAvailable?: number; dockerVersion?: string; dockerRunning?: boolean; directoriesWritable?: boolean; runtimeImages?: string[]; libraries?: HostLibrary[]; currentOperation?: string | null; [key: string]: unknown; + capabilities?: string[]; } export interface Host { id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number; @@ -23,6 +38,7 @@ export interface Host { lastHeartbeat: string | null; connected: boolean; connectionState: 'online' | 'offline'; readiness: HostReadiness; readinessReasons: string[]; inventory: HostInventory; enrolled: boolean; credentialActive: boolean; + decommission: HostDecommission | null; } export interface HostOperation { id: string; hostId: string; kind: string; instanceId: string | null; generation: number | null; diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts index f0329a1..8a16d59 100644 --- a/src/lib/server/hosts/index.ts +++ b/src/lib/server/hosts/index.ts @@ -1,8 +1,8 @@ -import { createHash, randomBytes, randomUUID } from 'node:crypto'; +import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'; import { isIP } from 'node:net'; import { db } from '../db/index.js'; import { ensureHostSchema } from './schema.js'; -import type { Host, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js'; +import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js'; export const AGENT_PROTOCOL_VERSION = 1; export const HEARTBEAT_MAX_AGE_MS = 90_000; @@ -11,6 +11,14 @@ const digest = (token: string) => createHash('sha256').update(token).digest('hex function database() { const handle = db(); ensureHostSchema(handle); return handle; } type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number }; const hostSql = `SELECT h.*, EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id = h.id AND c.revoked_at IS NULL) AS credential_active FROM ec_hosts h`; +type DecommissionRow = { host_id: string; operation_id: string; token_hash: string; stage: HostDecommission['stage']; requested_at: string; updated_at: string; cleaned_at: string | null; error: string | null }; +function decommissionRow(hostId: string): DecommissionRow | undefined { + return database().prepare('SELECT * FROM ec_host_decommissions WHERE host_id=?').get(hostId) as DecommissionRow | undefined; +} +function presentDecommission(row: DecommissionRow | undefined): HostDecommission | null { + return row ? { operationId: row.operation_id, stage: row.stage, requestedAt: row.requested_at, updatedAt: row.updated_at, + error: row.error, retryable: row.stage === 'failed' && getHostOperation(row.operation_id)?.state === 'failed' } : null; +} function hostFromRow(row: HostRow): Host { const inventory: HostInventory = JSON.parse(row.inventory); const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS; @@ -29,11 +37,13 @@ function hostFromRow(row: HostRow): Host { if (!libraryReady) reasons.push('Download and verify a game-file library.'); if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.'); if (row.mode === 'disabled') reasons.push('Host is disabled.'); + const decommission = presentDecommission(decommissionRow(row.id)); + if (decommission) reasons.push('Host deletion is in progress; waiting for confirmed cleanup and agent removal.'); return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max, bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes, createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline', - readiness: row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready', - readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active }; + readiness: decommission ? 'deleting' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready', + readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active, decommission }; } export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); } export function getHost(id: string): Host | null { @@ -43,14 +53,18 @@ export function getHost(id: string): Host | null { function requireHost(id: string): Host { const host = getHost(id); if (!host) throw new Error('Host was not found.'); return host; } export class HostDeletionError extends Error { - constructor(message: string, readonly status: 404 | 409, readonly blockers: string[] = []) { super(message); } + constructor(message: string, readonly status: 400 | 401 | 404 | 409, readonly blockers: string[] = []) { super(message); } } /** Controller records can be removed only after all remote instance ownership is resolved. */ export function hostDeletionStatus(id: string): HostDeletion { - if (!getHost(id)) throw new HostDeletionError('Host not found.', 404); + const host = getHost(id); + if (!host) throw new HostDeletionError('Host not found.', 404); + if (host.decommission && !host.decommission.retryable) return { allowed: true, blockers: [] }; const handle = database(); const blockers: string[] = []; + if (host.enrolled && !host.inventory.capabilities?.includes('host.decommission')) blockers.push('Update this host agent before deleting it: generate an installer from Agent access, rerun it on this host, and wait for a fresh heartbeat.'); + if (host.enrolled && !host.credentialActive) blockers.push('Re-enroll this host so its agent can receive and complete the cleanup request.'); const instances = handle.prepare('SELECT server FROM native_instances WHERE host_id=? AND retired_at IS NULL ORDER BY server').all(id) as { server: string }[]; if (instances.length) { const names = instances.slice(0, 5).map(instance => instance.server).join(', ') + (instances.length > 5 ? ', …' : ''); @@ -70,20 +84,68 @@ export function hostDeletionStatus(id: string): HostDeletion { return { allowed: blockers.length === 0, blockers }; } -/** Retain references and audit history; deletion never sends remote cleanup commands. */ -export function deleteHost(id: string): void { +/** Keep the host visible and authenticated until its independent cleanup worker proves removal. */ +export function deleteHost(id: string): { deleted: boolean; operationId?: string } { const handle = database(); - handle.transaction(() => { + return handle.transaction(() => { + const host = getHost(id); + if (!host) throw new HostDeletionError('Host not found.', 404); + if (host.decommission && !host.decommission.retryable) return { deleted: false, operationId: host.decommission.operationId }; const status = hostDeletionStatus(id); if (!status.allowed) throw new HostDeletionError(status.blockers.join(' '), 409, status.blockers); const timestamp = now(); - const reason = 'Host deleted before this operation was delivered to the agent.'; + const reason = 'Host deletion requested before this operation was delivered to the agent.'; handle.prepare(`INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) SELECT id,'failed',?,? FROM ec_host_operations WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL`).run(timestamp, reason, id); handle.prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL").run(reason, timestamp, id); - handle.prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(timestamp, id); handle.prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, id); - handle.prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL").run(timestamp, id); + if (!host.enrolled) { + handle.prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL").run(timestamp, id); + return { deleted: true }; + } + const operationId = randomUUID(); + const token = 'ec_cleanup_' + randomBytes(32).toString('base64url'); + const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` }; + handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at) VALUES(?,?,'host.decommission',?,'pending',?,?)").run(operationId, id, JSON.stringify(payload), timestamp, timestamp); + handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp); + handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?) + ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`) + .run(id, operationId, digest(token), timestamp, timestamp); + handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id); + return { deleted: false, operationId }; + }).immediate(); +} + +export function receiveDecommissionReceipt(operationId: string, token: string, input: Record): { acknowledged: true } { + if (!token || token.length > 200) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401); + return database().transaction(() => { + const row = database().prepare('SELECT * FROM ec_host_decommissions WHERE operation_id=?').get(operationId) as DecommissionRow | undefined; + if (!row || !timingSafeEqual(Buffer.from(row.token_hash, 'hex'), Buffer.from(digest(token), 'hex'))) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401); + if (!['cleaned', 'completed', 'failed'].includes(input.stage as string)) throw new HostDeletionError('Invalid cleanup receipt stage.', 400); + if (row.stage === 'completed') return { acknowledged: true as const }; + const operation = getHostOperation(operationId)!; + if (!['accepted','running','unknown'].includes(operation.state)) throw new HostDeletionError('Cleanup has not been accepted by the host.', 409); + const timestamp = now(); + if (input.stage === 'failed') { + database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE host_id=?") + .run(typeof input.error === 'string' ? input.error.slice(0, 2000) : 'Host cleanup needs attention. The cleanup worker will retry.', timestamp, row.host_id); + return { acknowledged: true as const }; + } + if (input.stage === 'cleaned') { + database().prepare("UPDATE ec_host_decommissions SET stage='cleaned',cleaned_at=COALESCE(cleaned_at,?),error=NULL,updated_at=? WHERE host_id=?").run(timestamp, timestamp, row.host_id); + return { acknowledged: true as const }; + } + if (!row.cleaned_at) throw new HostDeletionError('The host must confirm file cleanup before confirming agent removal.', 409); + database().prepare("UPDATE ec_host_decommissions SET stage='completed',error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id); + database().prepare("UPDATE ec_host_operations SET state='completed',result=?,error=NULL,updated_at=? WHERE id=?").run(JSON.stringify({ filesRemoved: true, agentRemoved: true }), timestamp, operationId); + database().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'completed',?)").run(operationId, timestamp); + database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(timestamp, row.host_id); + database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, row.host_id); + if (database().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='host_library_caches'").get()) { + database().prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id); + } + database().prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=?").run(timestamp, row.host_id); + return { acknowledged: true as const }; }).immediate(); } function textField(value: unknown, label: string, max = 200): string { @@ -102,6 +164,7 @@ export function createHost(input: Record): Host { } export function updateHost(id: string, input: Record): Host { const host = requireHost(id); + if (host.decommission) throw new Error('Host cleanup is in progress; its settings cannot change.'); const name = input.name === undefined ? host.name : textField(input.name, 'Host name'); const mode = input.mode ?? host.mode; if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.'); @@ -117,7 +180,8 @@ export function updateHost(id: string, input: Record): Host { } export function createEnrollment(hostId: string, ttlMs = 15 * 60_000): { id: string; hostId: string; token: string; expiresAt: string } { - requireHost(hostId); + const host = requireHost(hostId); + if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup is in progress; enrollment is unavailable.'); if (!Number.isInteger(ttlMs) || ttlMs <= 0 || ttlMs > 60 * 60_000) throw new Error('Enrollment lifetime must be at most one hour.'); const token = 'ec_enroll_' + randomBytes(32).toString('base64url'); const id = randomUUID(); const expiresAt = new Date(Date.now() + ttlMs).toISOString(); @@ -143,6 +207,7 @@ export function validateInventory(value: unknown): HostInventory { } for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`); if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.'); + if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.'); if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library => !library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.'); return inventory; @@ -169,7 +234,8 @@ export function authenticateCredential(token: string): Host | null { return getHost(row.host_id); } export function revokeCredentials(hostId: string): void { - requireHost(hostId); + const host = requireHost(hostId); + if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup needs its credentials until removal is confirmed.'); database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(now(), hostId); } export function rotateCredential(hostId: string, nextCredential?: string): string { @@ -207,6 +273,8 @@ export function getHostOperation(id: string): HostOperation | null { const row = export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); } export function queueHostOperation(hostId: string, kind: string, payload: Record, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation { const host = requireHost(hostId); + if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.'); + if (kind === 'host.decommission') throw new Error('Use the host deletion workflow to request cleanup.'); if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.'); if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.'); if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.'); @@ -245,6 +313,7 @@ export function reportHostOperation(hostId: string, id: string, input: { state: requireHost(hostId); const operation = getHostOperation(id); if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.'); + if (operation.kind === 'host.decommission' && input.state === 'completed') throw new Error('Host removal requires the cleanup worker completion receipt.'); if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.'); if (['completed','failed'].includes(operation.state)) return operation; const state = input.state as HostOperationState; @@ -253,6 +322,10 @@ export function reportHostOperation(hostId: string, id: string, input: { state: const result = input.result === undefined ? null : JSON.stringify(input.result); if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.'); database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=? WHERE id=?').run(state, result, error, now(), id); + if (operation.kind === 'host.decommission') { + if (state === 'failed') database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?").run(error ?? 'The agent could not start cleanup. Repair the installation, then retry deletion.', now(), id); + else database().prepare("UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')").run(now(), id); + } if (operation.state !== state) database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)').run(id, state, now(), error); return getHostOperation(id)!; })(); @@ -270,6 +343,7 @@ export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean { const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress; + if (host.decommission) return false; if (candidate < host.portMin || candidate > host.portMax) return false; const rows = database().prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND protocol=? AND port=?').all(hostId, protocol, candidate) as { instance_id: string; bind_address: string }[]; return !rows.some(row => row.instance_id !== instanceId && (row.bind_address === bind || ['0.0.0.0','::'].includes(bind) || ['0.0.0.0','::'].includes(row.bind_address))); @@ -281,6 +355,7 @@ export function nextHostPort(hostId: string, additionalPorts: number[] = [], bin } export function reserveHostPort(hostId: string, instanceId: string, options: { port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): number { const host = requireHost(hostId); const bind = options.bindAddress ?? host.bindAddress; const protocol = options.protocol ?? 'udp'; + if (host.decommission) throw new Error('Host cleanup is in progress; new port reservations are blocked.'); if (!isIP(bind)) throw new Error('Bind address must be an IP address.'); if (!['udp','tcp'].includes(protocol)) throw new Error('Port protocol must be UDP or TCP.'); if (!instanceId) throw new Error('An immutable instance ID is required.'); diff --git a/src/lib/server/hosts/schema.ts b/src/lib/server/hosts/schema.ts index 6abb3ab..ef50965 100644 --- a/src/lib/server/hosts/schema.ts +++ b/src/lib/server/hosts/schema.ts @@ -40,6 +40,13 @@ export function ensureHostSchema(handle: Database.Database): void { PRIMARY KEY(host_id,bind_address,protocol,port) ); CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id); + CREATE TABLE IF NOT EXISTS ec_host_decommissions ( + host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id), + operation_id TEXT NOT NULL UNIQUE REFERENCES ec_host_operations(id), + token_hash TEXT NOT NULL, stage TEXT NOT NULL DEFAULT 'queued' + CHECK(stage IN ('queued','running','cleaned','failed','completed')), + requested_at TEXT NOT NULL, updated_at TEXT NOT NULL, cleaned_at TEXT, error TEXT + ); `); const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[]; if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT'); diff --git a/src/lib/ui/components/DeleteHostDialog.svelte b/src/lib/ui/components/DeleteHostDialog.svelte index 4703575..428bd4a 100644 --- a/src/lib/ui/components/DeleteHostDialog.svelte +++ b/src/lib/ui/components/DeleteHostDialog.svelte @@ -4,7 +4,7 @@ import Dialog from './primitives/Dialog.svelte'; let { open = $bindable(false), host, ondeleted }: - { open?: boolean; host: Pick | null; ondeleted: () => void | Promise } = $props(); + { open?: boolean; host: Pick | null; ondeleted: (result: { deleted: boolean; operationId?: string }) => void | Promise } = $props(); let deletion = $state(null); let checking = $state(false); let deleting = $state(false); let error = $state(''); @@ -36,19 +36,20 @@ throw new Error(body.error ?? body.message ?? 'Could not delete this host.'); } open = false; - await ondeleted(); + await ondeleted(body); } catch (cause) { error = (cause as Error).message; } finally { deleting = false; } } + description="Permanently remove Electric Cherry's files from this host, including its downloaded game files, then uninstall the agent.">
-

The agent and game files stay on the machine. Published game libraries and operation history are kept. Commands that have never been sent to the host will be cancelled.

+

This deletes the host's game-library copies, server files, agent data, credentials, and service installation. Electric Cherry's uploaded libraries on the controller and its operation history are kept.

+

The host remains visible until cleanup is confirmed. Offline hosts wait for reconnection. Unsent commands will be cancelled.

{#if checking}

Checking servers and unfinished operations…

{:else if deletion && !deletion.allowed}
This host still has work to finish:
    {#each deletion.blockers as blocker}
  • {blocker}
    • {/each}

Open open = false}>host details to review its operations or drain its servers.

- {:else if deletion?.allowed}

This host is ready to delete. Adding it again requires a new enrollment.

{/if} + {:else if deletion?.allowed}

Cleanup cannot be undone once it starts. Adding this machine again requires installing and enrolling the agent again.

{/if} {#if error}{/if}
{#snippet actions()} diff --git a/src/routes/api/agent/decommission/[id]/+server.ts b/src/routes/api/agent/decommission/[id]/+server.ts new file mode 100644 index 0000000..00bdd10 --- /dev/null +++ b/src/routes/api/agent/decommission/[id]/+server.ts @@ -0,0 +1,14 @@ +import { json } from '@sveltejs/kit'; +import { HostDeletionError, receiveDecommissionReceipt } from '$lib/server/hosts/index.js'; +import { objectBody, requireAgentTransport } from '$lib/server/hosts/http.js'; + +/** The uninstall worker retains only this narrowly scoped credential after removing the agent. */ +export async function POST({ request, params }: { request: Request; params: { id: string } }) { + requireAgentTransport(request); + const token = request.headers.get('authorization')?.match(/^Bearer ([A-Za-z0-9_-]+)$/i)?.[1] ?? ''; + try { return json(receiveDecommissionReceipt(params.id, token, await objectBody(request, 8192)), { headers: { 'Cache-Control': 'no-store' } }); } + catch (cause) { + if (cause instanceof HostDeletionError) return json({ error: cause.message }, { status: cause.status }); + throw cause; + } +} diff --git a/src/routes/api/hosts/[id]/+server.ts b/src/routes/api/hosts/[id]/+server.ts index 7e7523c..a210d6b 100644 --- a/src/routes/api/hosts/[id]/+server.ts +++ b/src/routes/api/hosts/[id]/+server.ts @@ -15,7 +15,7 @@ export async function PATCH({ request, params }: Event) { export function DELETE({ request, params }: Event) { requireManagementOrigin(request); - try { deleteHost(params.id); return json({ deleted: true }); } + try { const result = deleteHost(params.id); return json(result, { status: result.deleted ? 200 : 202 }); } catch (cause) { if (cause instanceof HostDeletionError) return json({ error: cause.message, blockers: cause.blockers }, { status: cause.status }); return hostApiError(cause); diff --git a/src/routes/hosts/+page.svelte b/src/routes/hosts/+page.svelte index 44bbc56..49ab66c 100644 --- a/src/routes/hosts/+page.svelte +++ b/src/routes/hosts/+page.svelte @@ -2,7 +2,7 @@ import { onMount } from 'svelte'; import { page } from '$app/stores'; import type { Host } from '$lib/hosts.js'; - import { hostReadinessLabel } from '$lib/hosts.js'; + import { hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Button from '$ui/components/primitives/Button.svelte'; import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte'; @@ -18,8 +18,8 @@ try { const response = await fetch('/api/hosts'); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); if (sequence === loadSequence) { hosts = body.hosts; error = ''; } } catch (cause) { if (sequence === loadSequence) error = (cause as Error).message; } finally { if (sequence === loadSequence) loading = false; } } - async function deleted() { - notice = `${deletingHost?.name ?? 'Host'} deleted.`; + async function deleted(result: { deleted: boolean }) { + notice = result.deleted ? `${deletingHost?.name ?? 'Host'} deleted.` : `Cleanup requested for ${deletingHost?.name ?? 'host'}. It remains listed until its files and agent are removed.`; if (enrollment?.hostId === deletingHost?.id) enrollment = null; deletingHost = null; await load(); @@ -51,7 +51,7 @@ {#if enrollment}

Install the agent

This single-use command expires {new Date(enrollment.expiresAt).toLocaleString()}. Run it with administrator access on your game host.

{enrollment.command}
Open host setup →
{/if} {:else}

Administrators manage host setup, runtime preparation, and credentials.

{/if} {#if loading}

Loading hosts…

{:else if !hosts.length}

No hosts connected yet.{canManage ? ' Add a host to begin.' : ''}

{:else} -
{#each hosts as host (host.id)}

{host.name}

{hostReadinessLabel(host.readiness)}

{host.hostname || 'Awaiting enrollment'} · {host.connectionState}

{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}

Game ports {host.portMin}–{host.portMax}

{#if host.readinessReasons.length}

{host.readinessReasons[0]}

{/if}
{#if host.readiness === 'needs-game-files'}{/if}{#if canManage}{/if}
{/each}
+
{#each hosts as host (host.id)}

{host.name}

{hostReadinessLabel(host.readiness)}

{host.hostname || 'Awaiting enrollment'} · {host.connectionState}

{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}

Game ports {host.portMin}–{host.portMax}

{#if host.decommission}

{hostDeletionMessage(host)}

{#if host.decommission.error}

{host.decommission.error}

{/if}{:else if host.readinessReasons.length}

{host.readinessReasons[0]}

{/if}
{#if host.readiness === 'needs-game-files'}{/if}{#if canManage && (!host.decommission || host.decommission.retryable)}{/if}
{/each}
{/if} diff --git a/src/routes/hosts/[id]/+page.svelte b/src/routes/hosts/[id]/+page.svelte index 2544cdb..e393a79 100644 --- a/src/routes/hosts/[id]/+page.svelte +++ b/src/routes/hosts/[id]/+page.svelte @@ -3,7 +3,7 @@ import { page } from '$app/stores'; import { goto } from '$app/navigation'; import type { Host, HostOperation, HostMode } from '$lib/hosts.js'; - import { hostReadinessLabel } from '$lib/hosts.js'; + import { hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Button from '$ui/components/primitives/Button.svelte'; import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte'; @@ -14,10 +14,11 @@ let runtimeImage = $state('ghcr.io/ayymoss/plutainer:latest'); let enrollment = $state<{ command: string; expiresAt: string } | null>(null); const endpoint = $derived(`/api/hosts/${$page.params.id}`); const canManage = $derived($page.data.user?.role === 'admin'); + const cleaning = $derived(!!host?.decommission); function bytes(value: unknown) { return typeof value === 'number' ? `${(value / 1073741824).toFixed(1)} GiB` : 'Unknown'; } async function load(initialize = false) { try { - const response = await fetch(endpoint); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations; + const response = await fetch(endpoint); const body = await response.json(); if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations; if (initialize && host) { name = host.name; portMin = host.portMin; portMax = host.portMax; bindAddress = host.bindAddress; publicAddress = host.publicAddress; minimumFreeGB = host.minimumFreeBytes / 1073741824; } } catch (cause) { error = (cause as Error).message; } finally { loading = false; } } @@ -33,30 +34,31 @@ {host?.name ?? 'Host'} · Electric Cherry
← All hosts - {#snippet actions()}{#if canManage && host}{/if}{/snippet} + {#snippet actions()}{#if canManage && host && (!cleaning || host.decommission?.retryable)}{/if}{/snippet} {#if error}{/if}{#if notice}

{notice}

{/if} {#if loading}

Loading host…

{:else if host} + {#if host.decommission}

Host cleanup

{hostDeletionMessage(host)}

{#if host.decommission.error}

{host.decommission.error}

{/if}

Requested {new Date(host.decommission.requestedAt).toLocaleString()}. This page closes only after the cleanup worker confirms removal.

{#if host.decommission.stage === 'cleaned'}

The agent is expected to go offline during uninstall. If final confirmation never arrives, inspect the machine; loss of contact alone does not prove completion.

{/if}
{/if}

{host.connectionState} · {hostReadinessLabel(host.readiness)}

Last heartbeat: {host.lastHeartbeat ? new Date(host.lastHeartbeat).toLocaleString() : 'Never'}
{#if host.readinessReasons.length}
    {#each host.readinessReasons as reason}
  • {reason}
    • {/each}
{:else}

Connected and ready. Each deployment also checks its game library, runtime, and ports.

{/if} {#if host.readiness === 'needs-game-files'}

Choose a published game-file version and download it to this host →

{/if} - {#if canManage}
{/if} + {#if canManage && !cleaning}
{/if}

Draining prevents new deployments and retires instances as they become safe to remove. Disabled hosts receive no new commands; running game containers continue.

Inventory

Hostname
{host.hostname || 'Pending'}
Agent
{host.inventory.agentVersion ?? 'Unknown'}
Platform
{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}
CPU
{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)
Memory available / total
{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}
Game storage available / total
{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}
Docker
{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}
Current operation
{host.inventory.currentOperation ?? 'None reported'}
-

Host configuration

{#if canManage}
+

Host configuration

{#if canManage}
{:else}
Friendly name
{host.name}
Game ports
{host.portMin}–{host.portMax}
Local bind address
{host.bindAddress}
Public / NAT address
{host.publicAddress || 'Not configured'}
Minimum free storage
{bytes(host.minimumFreeBytes)}

Host configuration is managed by administrators.

{/if}

Public addressing is separate from local port reservations. Configure your firewall and NAT for the game-port range.

Game files and runtime

Manage game-file libraries →

{#if host.inventory.libraries?.length}
    {#each host.inventory.libraries as library}
  • {library.id}: {library.state}{library.progress !== undefined ? ` (${library.progress}%)` : ''}
    • {/each}
{:else}

No cached game libraries reported.

{/if} - {#if canManage}
{ event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}>
{/if}

Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported'}

-

Agent access

Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.

{#if canManage}
+ {#if canManage}
{ event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}>
{/if}

Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported'}

+

Agent access

Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.

{#if canManage}
{#if enrollment}

Expires {new Date(enrollment.expiresAt).toLocaleString()}

{enrollment.command}
{/if}{:else}

Administrators manage enrollment and agent credentials.

{/if}

Operation history

{#if !operations.length}

No remote operations yet.

{:else}
{#each operations as operation}{/each}
OperationStateUpdatedResult
{operation.kind}{operation.id}{operation.state}{new Date(operation.updatedAt).toLocaleString()}{operation.error ?? '—'}
{/if}
{/if}
- goto('/hosts')} /> + { if (result.deleted) await goto('/hosts'); else { enrollment = null; await load(); } }} />