Compare commits
4
Commits
95cf7dd36f
...
020894b816
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
020894b816 | ||
|
|
62d7817ea3 | ||
|
|
3d62dfff27 | ||
|
|
840f02b53e |
@@ -29,11 +29,13 @@ The default Compose data mount preserves the previous
|
||||
`EC_DATA_DIR` to your preferred persistent directory. Back up that directory,
|
||||
including SQLite metadata and published game-file storage.
|
||||
|
||||
1. **Hosts → Add Host**: configure a Debian 12/13 amd64 machine and run its
|
||||
1. **Library → Game Files**: select T5 or T6 and upload the complete local game
|
||||
folder, including its DLC files. Name the library and version, then publish it.
|
||||
Uploads work before any host is enrolled.
|
||||
2. **Hosts → Add Host**: configure a Debian 12/13 amd64 machine and run its
|
||||
generated enrollment command.
|
||||
2. Prepare a Plutainer runtime from the host page.
|
||||
3. **Library → Game Files**: import existing T5/T6 files once, then cache a
|
||||
published version on the destination host.
|
||||
3. Prepare a Plutainer runtime from the host page and download the published
|
||||
game-file version to that host from **Game Files**.
|
||||
4. Add the exact Plutonium server keys in **Keys**, or supply missing keys when
|
||||
launching.
|
||||
5. Launch a server or collection with an explicit host, game-file version and
|
||||
@@ -69,9 +71,12 @@ server ID and retained after removal.
|
||||
|
||||
## Libraries and campaigns
|
||||
|
||||
Game-file versions include game, version, DLC metadata, manifests, checksums and
|
||||
size. Actual game files use dedicated filesystem storage, never SQLite blobs.
|
||||
Downloads resume, report progress and verify checksums before becoming usable.
|
||||
Game-file versions contain the uploaded T5/T6 folder, with its directory layout
|
||||
and included DLC preserved. There is no separate DLC declaration. Versions record
|
||||
the game, label, file manifest, checksums and size. Uploads use bounded chunks,
|
||||
support pause/resume and verify checksums before publication. Actual game files
|
||||
use dedicated filesystem storage, never SQLite blobs. Host downloads resume,
|
||||
report progress and verify checksums before becoming usable.
|
||||
Multiple instances on one host share read-only base files.
|
||||
|
||||
Scripts, plugins, configuration baselines and overlays retain their versioning,
|
||||
|
||||
+338
-4
@@ -29,10 +29,17 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.0.0"
|
||||
VERSION = "1.1.0"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
|
||||
AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
|
||||
AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
|
||||
AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service")
|
||||
UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
|
||||
UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service")
|
||||
UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
|
||||
ROOT_MARKER = ".electric-cherry.json"
|
||||
|
||||
|
||||
class AgentError(Exception):
|
||||
@@ -47,6 +54,97 @@ class Rejected(AgentError):
|
||||
"""A definitive precondition failure, before any destructive action."""
|
||||
|
||||
|
||||
def systemctl(*args):
|
||||
try:
|
||||
result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
raise AgentError("Host service management is unavailable") from None
|
||||
if result.returncode:
|
||||
raise AgentError("Host service management did not complete")
|
||||
|
||||
|
||||
def protected_path(path, missing=False):
|
||||
"""Deletion authority must never come from a runtime-writable path."""
|
||||
path = Path(path)
|
||||
for current in (*reversed(path.parents), path):
|
||||
try:
|
||||
info = current.lstat()
|
||||
except FileNotFoundError:
|
||||
if missing:
|
||||
continue
|
||||
raise AgentError("Host uninstall files are unavailable; reinstall the agent") from None
|
||||
if stat_module.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
|
||||
raise AgentError("Host uninstall paths must be owned and writable only by root")
|
||||
return path
|
||||
|
||||
|
||||
def protected_file(path, missing=False):
|
||||
path = protected_path(path, missing=missing)
|
||||
try:
|
||||
if not stat_module.S_ISREG(path.lstat().st_mode):
|
||||
raise AgentError("Host uninstall metadata must be a regular file")
|
||||
except FileNotFoundError:
|
||||
if not missing:
|
||||
raise
|
||||
return path
|
||||
|
||||
|
||||
def sync_parent(path):
|
||||
descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def uninstall_root(value, identity=None):
|
||||
"""Accept only a canonical, dedicated directory with no nested mounts."""
|
||||
if not isinstance(value, str) or not re.fullmatch(r"/[A-Za-z0-9/_.-]+", value):
|
||||
raise AgentError("Host data directory is unsafe to remove")
|
||||
root = Path(value)
|
||||
forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"}
|
||||
if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden:
|
||||
raise AgentError("Host data directory is unsafe to remove")
|
||||
for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT):
|
||||
if installed == root or root in installed.parents:
|
||||
raise AgentError("Host data directory overlaps the agent installation")
|
||||
protected_path(root, missing=True)
|
||||
if root.exists():
|
||||
info = root.stat()
|
||||
if not stat_module.S_ISDIR(info.st_mode):
|
||||
raise AgentError("Host data directory is not a directory")
|
||||
if identity and [info.st_dev, info.st_ino] != identity:
|
||||
raise AgentError("Host data directory changed since deletion was authorized")
|
||||
try:
|
||||
# ProtectSystem/ReadWritePaths give the running agent a private bind
|
||||
# mount at its data directory. Inspect the host systemd namespace,
|
||||
# otherwise every legitimate installation appears to be a mount root.
|
||||
mount_lines = Path("/proc/1/mountinfo").read_text().splitlines()
|
||||
except FileNotFoundError:
|
||||
mount_lines = [] # Non-Linux tests; the installed service requires Linux.
|
||||
for line in mount_lines:
|
||||
fields = line.split()
|
||||
if len(fields) < 5:
|
||||
continue
|
||||
mount = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), fields[4])
|
||||
if mount == value or mount.startswith(value + "/"):
|
||||
raise AgentError("Unmount filesystems inside the host data directory before deletion")
|
||||
return root
|
||||
|
||||
|
||||
def uninstall_ownership(config):
|
||||
root = uninstall_root(config.get("root"))
|
||||
marker = protected_file(root / ROOT_MARKER)
|
||||
try:
|
||||
saved = json.loads(marker.read_text())
|
||||
except (OSError, ValueError):
|
||||
raise AgentError("Host data ownership is unavailable; reinstall the agent") from None
|
||||
expected = {"version": 1, "root": str(root), "hostId": config.get("hostId"), "controller": config.get("controller", "").rstrip("/")}
|
||||
if saved != expected or not expected["hostId"]:
|
||||
raise AgentError("Host data ownership does not match this enrollment")
|
||||
return root
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
@@ -275,7 +373,7 @@ class Docker:
|
||||
raise AgentError("Docker command unavailable or timed out") from None
|
||||
if result.returncode:
|
||||
# Never include arbitrary Docker stderr: create errors may echo secrets.
|
||||
if tolerate_missing and (b"No such object" in (result.stderr or b"") or b"No such container" in (result.stderr or b"")):
|
||||
if tolerate_missing and any(message in (result.stderr or b"") for message in (b"No such object", b"No such container", b"No such image")):
|
||||
return None
|
||||
raise AgentError("Docker rejected the requested operation")
|
||||
return result.stdout.decode("utf-8", "replace")
|
||||
@@ -285,6 +383,188 @@ class Docker:
|
||||
return json.loads(raw)[0] if raw is not None else None
|
||||
|
||||
|
||||
class Uninstaller:
|
||||
"""A separate systemd service owns cleanup after stopping the host agent.
|
||||
|
||||
Its root-only request and executable stay on disk until the controller
|
||||
acknowledges data removal. Completion is reported only after removing those
|
||||
final files. A lost final response can be retried from memory; a power loss
|
||||
in that last window deliberately leaves controller status unconfirmed.
|
||||
"""
|
||||
|
||||
def __init__(self, request_path, docker=None):
|
||||
if Path(request_path) != UNINSTALL_REQUEST:
|
||||
raise AgentError("Invalid host uninstall request location")
|
||||
protected_file(UNINSTALL_REQUEST)
|
||||
self.job = json.loads(UNINSTALL_REQUEST.read_text())
|
||||
self.docker = docker or Docker()
|
||||
self.root = uninstall_root(self.job.get("root"), self.job.get("rootIdentity"))
|
||||
host_id = identifier(self.job.get("hostId"))
|
||||
operation_id = identifier(self.job.get("operationId"))
|
||||
parsed = urllib.parse.urlsplit(self.job.get("controller", ""))
|
||||
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise AgentError("Host uninstall controller must use HTTPS")
|
||||
if self.job.get("receiptUrl") != "/api/agent/decommission/" + operation_id or not isinstance(self.job.get("receiptToken"), str) or not self.job["receiptToken"]:
|
||||
raise AgentError("Invalid host uninstall receipt")
|
||||
if not isinstance(self.job.get("rootIdentity"), list) or len(self.job["rootIdentity"]) != 2 or not all(isinstance(value, int) and not isinstance(value, bool) for value in self.job["rootIdentity"]):
|
||||
raise AgentError("Invalid host data ownership proof")
|
||||
if self.root.exists():
|
||||
# The authorized device/inode is persisted before deletion so a
|
||||
# crash halfway through rmtree does not require a deleted marker.
|
||||
if not self.job.get("removalStarted"):
|
||||
uninstall_ownership({"root": str(self.root), "controller": self.job["controller"], "hostId": host_id})
|
||||
|
||||
def save(self, **changes):
|
||||
self.job.update(changes)
|
||||
atomic_json(UNINSTALL_REQUEST, self.job)
|
||||
|
||||
def receipt(self, stage, error=None):
|
||||
payload = {"stage": stage}
|
||||
if error:
|
||||
payload["error"] = error
|
||||
request = urllib.request.Request(self.job["controller"].rstrip("/") + self.job["receiptUrl"], data=json.dumps(payload).encode(), headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.job["receiptToken"], "User-Agent": "electric-cherry-agent/" + VERSION}, method="POST")
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, hdrs, newurl):
|
||||
raise AgentError("Controller redirects are not allowed")
|
||||
try:
|
||||
with urllib.request.build_opener(NoRedirect()).open(request, timeout=35) as response:
|
||||
if json.load(response).get("acknowledged") is not True:
|
||||
raise AgentError("Controller has not acknowledged host removal")
|
||||
except (OSError, ValueError):
|
||||
raise AgentError("Host removal receipt is waiting for controller contact") from None
|
||||
|
||||
def containers(self):
|
||||
ids = self.docker.call("container", "ls", "--all", "--quiet", "--no-trunc").splitlines()
|
||||
values = []
|
||||
for container_id in ids:
|
||||
info = self.docker.inspect(container_id)
|
||||
if info:
|
||||
values.append(info)
|
||||
return values
|
||||
|
||||
def owned_container(self, info):
|
||||
labels = info.get("Config", {}).get("Labels", {}) or {}
|
||||
if labels.get(LABEL + ".managed") != "true":
|
||||
return False
|
||||
if labels.get(LABEL + ".host") not in (None, self.job["hostId"]):
|
||||
return False
|
||||
try:
|
||||
instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", "")))
|
||||
except (AgentError, ValueError, TypeError):
|
||||
return False
|
||||
expected = str(self.root / "instances" / instance_id / str(generation) / "app")
|
||||
# Legacy agents did not label the host; its exact managed bind mount
|
||||
# proves ownership without touching another installation's containers.
|
||||
return any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", []))
|
||||
|
||||
def uses_root(self, info):
|
||||
for mount in info.get("Mounts", []):
|
||||
if mount.get("Type") != "bind":
|
||||
continue
|
||||
source = Path(mount.get("Source", "/")).resolve()
|
||||
if source == self.root or self.root in source.parents:
|
||||
return True
|
||||
return False
|
||||
|
||||
def remove_docker_resources(self):
|
||||
containers = self.containers()
|
||||
if any(self.uses_root(info) and not self.owned_container(info) for info in containers):
|
||||
raise AgentError("An unrelated container uses the host data directory; detach it before retrying deletion")
|
||||
owned = [info for info in containers if self.owned_container(info)]
|
||||
if any(info.get("State", {}).get("Running") is not False or info.get("State", {}).get("Status") not in ("created", "exited", "dead") for info in owned):
|
||||
raise AgentError("A managed game container is still running or its state is unknown; retire it before deleting this host")
|
||||
for info in owned:
|
||||
# Do not force removal: Docker must also reject a container that
|
||||
# starts again after inspection rather than interrupting players.
|
||||
self.docker.call("container", "rm", "--volumes", info["Id"], timeout=120, tolerate_missing=True)
|
||||
remaining = self.containers()
|
||||
if any(self.uses_root(info) for info in remaining):
|
||||
raise AgentError("Containers still use the host data directory")
|
||||
# Never prune Docker globally. Only remove this agent's recorded image
|
||||
# references, and preserve images used by any unrelated container.
|
||||
used_images = {info.get("Image") for info in remaining}
|
||||
images = self.job.get("runtimeImages", {})
|
||||
if not isinstance(images, dict):
|
||||
raise AgentError("Managed runtime image ownership is unavailable")
|
||||
for reference, expected_id in images.items():
|
||||
if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,511}", reference) or not isinstance(expected_id, str):
|
||||
raise AgentError("Managed runtime image ownership is invalid")
|
||||
raw = self.docker.call("image", "inspect", reference, tolerate_missing=True)
|
||||
if raw is None:
|
||||
continue
|
||||
found = json.loads(raw)
|
||||
if not found or found[0].get("Id") != expected_id or expected_id in used_images:
|
||||
continue
|
||||
self.docker.call("image", "rm", reference, tolerate_missing=True)
|
||||
|
||||
def remove_data(self):
|
||||
protected_path(AGENT_CONFIG.parent, missing=True)
|
||||
if not self.job.get("agentStopped"):
|
||||
systemctl("disable", "--now", AGENT_UNIT.name)
|
||||
self.save(agentStopped=True)
|
||||
self.remove_docker_resources()
|
||||
root = uninstall_root(str(self.root), self.job["rootIdentity"])
|
||||
# This checkpoint also authorizes safe continuation if a power loss
|
||||
# removes the ownership marker before the rest of the tree.
|
||||
self.save(removalStarted=True)
|
||||
if root.exists():
|
||||
if not shutil.rmtree.avoids_symlink_attacks:
|
||||
raise AgentError("This platform cannot safely remove managed host files")
|
||||
shutil.rmtree(root)
|
||||
sync_parent(root)
|
||||
for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT):
|
||||
protected_file(path, missing=True)
|
||||
path.unlink(missing_ok=True)
|
||||
sync_parent(path)
|
||||
systemctl("daemon-reload")
|
||||
self.save(dataRemoved=True)
|
||||
|
||||
def remove_helper(self):
|
||||
# Disable does not stop this process: it must outlive its own unit and
|
||||
# executable to deliver the final receipt after the files are gone.
|
||||
protected_file(UNINSTALL_UNIT, missing=True)
|
||||
if UNINSTALL_UNIT.exists():
|
||||
systemctl("disable", UNINSTALL_UNIT.name)
|
||||
UNINSTALL_UNIT.unlink()
|
||||
sync_parent(UNINSTALL_UNIT)
|
||||
systemctl("daemon-reload")
|
||||
protected_file(UNINSTALL_EXECUTABLE, missing=True)
|
||||
UNINSTALL_EXECUTABLE.unlink(missing_ok=True)
|
||||
sync_parent(UNINSTALL_EXECUTABLE)
|
||||
protected_path(UNINSTALL_REQUEST.parent, missing=True)
|
||||
if UNINSTALL_REQUEST.parent.exists():
|
||||
shutil.rmtree(UNINSTALL_REQUEST.parent)
|
||||
sync_parent(UNINSTALL_REQUEST.parent)
|
||||
|
||||
def run(self):
|
||||
if not self.job.get("dataRemoved"):
|
||||
try:
|
||||
self.remove_data()
|
||||
except (AgentError, OSError, ValueError, KeyError) as error:
|
||||
with contextlib.suppress(AgentError):
|
||||
self.receipt("failed", str(error) if isinstance(error, AgentError) else "Host cleanup could not finish. Check the uninstall service and resolve filesystem errors; cleanup will retry.")
|
||||
raise
|
||||
self.receipt("cleaned")
|
||||
while True:
|
||||
try:
|
||||
self.remove_helper()
|
||||
break
|
||||
except (AgentError, OSError, ValueError):
|
||||
# Once its unit/executable has been unlinked systemd can no
|
||||
# longer restart this worker. Finish idempotently in memory.
|
||||
with contextlib.suppress(AgentError):
|
||||
self.receipt("failed", "Host files were removed, but removing the uninstall service is still retrying")
|
||||
time.sleep(15)
|
||||
# The controller never marks a host deleted on the earlier receipt.
|
||||
# No credentials or retry state are left on disk at this point.
|
||||
while True:
|
||||
try:
|
||||
self.receipt("completed")
|
||||
return
|
||||
except AgentError:
|
||||
time.sleep(15)
|
||||
|
||||
|
||||
class Agent:
|
||||
def __init__(self, config, docker=None):
|
||||
self.config = config
|
||||
@@ -300,6 +580,43 @@ class Agent:
|
||||
self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
|
||||
self.executing_id = None
|
||||
|
||||
def can_uninstall(self):
|
||||
try:
|
||||
if Path(self.config_path) != AGENT_CONFIG:
|
||||
return False
|
||||
uninstall_ownership(self.config)
|
||||
protected_file(UNINSTALL_EXECUTABLE)
|
||||
protected_file(UNINSTALL_UNIT)
|
||||
return True
|
||||
except (AgentError, OSError, ValueError):
|
||||
return False
|
||||
|
||||
def decommission(self, operation):
|
||||
payload = operation.get("payload", {})
|
||||
if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]:
|
||||
raise Rejected("Host removal identity does not match this agent")
|
||||
if payload.get("receiptUrl") != "/api/agent/decommission/" + identifier(operation["id"]) or not isinstance(payload.get("receiptToken"), str) or not payload["receiptToken"]:
|
||||
raise Rejected("Invalid host removal receipt")
|
||||
if not self.can_uninstall():
|
||||
raise Rejected("Reinstall the agent to enable safe removal of its dedicated host files")
|
||||
root = uninstall_ownership(self.config)
|
||||
info = root.stat()
|
||||
protected_file(UNINSTALL_REQUEST, missing=True)
|
||||
aliases_path = root / "agent" / "runtime-images.json"
|
||||
protected_file(aliases_path, missing=True)
|
||||
aliases = json.loads(aliases_path.read_text()) if aliases_path.exists() else {}
|
||||
job = {"hostId": payload["hostId"], "operationId": operation["id"], "receiptUrl": payload["receiptUrl"], "receiptToken": payload["receiptToken"], "controller": self.config["controller"].rstrip("/"), "root": str(root), "rootIdentity": [info.st_dev, info.st_ino], "runtimeImages": aliases}
|
||||
if UNINSTALL_REQUEST.exists():
|
||||
previous = json.loads(UNINSTALL_REQUEST.read_text())
|
||||
if any(previous.get(key) != job[key] for key in ("hostId", "operationId", "receiptUrl", "receiptToken", "controller", "root", "rootIdentity")):
|
||||
raise Rejected("A different host removal is already pending")
|
||||
self.checkpoint({"phase": "uninstall-service"})
|
||||
if not UNINSTALL_REQUEST.exists():
|
||||
atomic_json(UNINSTALL_REQUEST, job)
|
||||
systemctl("enable", "--now", "--no-block", UNINSTALL_UNIT.name)
|
||||
self.stopping.set()
|
||||
return {"decommissioning": True, "message": "The uninstall service will report completion after host files and the agent are removed"}
|
||||
|
||||
def instance_dir(self, instance_id, generation):
|
||||
instance_id, generation = instance_identity(instance_id, generation)
|
||||
return self.root / "instances" / instance_id / str(generation)
|
||||
@@ -416,7 +733,7 @@ class Agent:
|
||||
for line in Path("/etc/os-release").read_text().splitlines():
|
||||
if line.startswith("PRETTY_NAME="):
|
||||
os_name = line.split("=", 1)[1].strip('"')
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
|
||||
def safe_to_disrupt(self, instance_id, generation):
|
||||
health = self.inspect(instance_id, generation)
|
||||
@@ -525,6 +842,8 @@ class Agent:
|
||||
name = self.container_name(instance_id, generation)
|
||||
spec_sha = digest(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode())
|
||||
args = ["container", "create", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)]
|
||||
if self.config.get("hostId"):
|
||||
args.extend(["--label", LABEL + ".host=" + identifier(self.config["hostId"])])
|
||||
bind_arg = "[" + bind + "]" if ":" in bind else bind
|
||||
for protocol in ("udp", "tcp"):
|
||||
args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
|
||||
@@ -923,6 +1242,8 @@ class Agent:
|
||||
kind, payload = operation.get("kind"), operation.get("payload", {})
|
||||
if not isinstance(kind, str) or not isinstance(payload, dict):
|
||||
raise AgentError("Invalid operation")
|
||||
if kind == "host.decommission":
|
||||
return self.decommission(operation)
|
||||
if kind == "credentials.rotate":
|
||||
pending = "ec_host_" + base64.urlsafe_b64encode(os.urandom(32)).rstrip(b"=").decode()
|
||||
self.config["pendingCredential"] = pending
|
||||
@@ -997,7 +1318,7 @@ class Agent:
|
||||
return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message)
|
||||
finally:
|
||||
self.executing_id = None
|
||||
return self.journal.update(operation["id"], "completed", result)
|
||||
return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result)
|
||||
|
||||
def recover_credential(self):
|
||||
pending = self.config.get("pendingCredential")
|
||||
@@ -1024,6 +1345,13 @@ class Agent:
|
||||
instance_id, generation = operation.get("instanceId"), operation.get("generation")
|
||||
checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
|
||||
try:
|
||||
if kind == "host.decommission":
|
||||
self.executing_id = operation["id"]
|
||||
try:
|
||||
result = self.decommission(operation)
|
||||
return self.journal.update(operation["id"], "running", result)
|
||||
finally:
|
||||
self.executing_id = None
|
||||
if kind == "credentials.rotate":
|
||||
if not self.config.get("pendingCredential") or self.recover_credential():
|
||||
return self.journal.update(operation["id"], "completed", {"rotated": True, "reconciled": True})
|
||||
@@ -1183,8 +1511,14 @@ def main():
|
||||
parser.add_argument("--token-file")
|
||||
parser.add_argument("--root", default="/var/lib/electric-cherry")
|
||||
parser.add_argument("--force-enroll", action="store_true")
|
||||
parser.add_argument("--uninstall", metavar="REQUEST", help="Run the installed host removal service")
|
||||
parser.add_argument("--version", action="version", version=VERSION)
|
||||
args = parser.parse_args()
|
||||
if args.uninstall:
|
||||
if os.geteuid() != 0 or args.enroll:
|
||||
raise AgentError("Host uninstall must run as the dedicated root service")
|
||||
Uninstaller(args.uninstall).run()
|
||||
return
|
||||
if args.enroll:
|
||||
if not args.controller or not args.token_file:
|
||||
parser.error("Enrollment needs --controller and --token-file")
|
||||
|
||||
+120
-7
@@ -21,7 +21,7 @@ done
|
||||
case "$controller" in https://*) ;; *) echo '--controller must be an HTTPS URL.' >&2; exit 2 ;; esac
|
||||
[ -n "$enrollment_token" ] || { echo '--token is required.' >&2; exit 2; }
|
||||
case "$data_dir" in /*) ;; *) echo '--data-dir must be absolute.' >&2; exit 2 ;; esac
|
||||
case "$data_dir" in *[!a-zA-Z0-9/_.-]*|/|/etc|/usr|/var|/var/lib) echo 'Choose a dedicated data directory.' >&2; exit 2 ;; esac
|
||||
case "$data_dir" in *[!a-zA-Z0-9/_.-]*|*/../*|*/./*|*/..|*/.|*//*|*/) echo 'Choose a canonical, dedicated data directory.' >&2; exit 2 ;; esac
|
||||
[ ! -L "$data_dir" ] || { echo 'Data directory must not be a symlink.' >&2; exit 2; }
|
||||
. /etc/os-release
|
||||
[ "${ID:-}" = debian ] || { echo 'Supported platform: Debian 12 or 13.' >&2; exit 1; }
|
||||
@@ -32,13 +32,82 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; }
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends ca-certificates curl python3
|
||||
python3 - "$controller" "$data_dir" <<'PY'
|
||||
import json, pathlib, sys
|
||||
config = pathlib.Path('/etc/electric-cherry/agent.json')
|
||||
if config.exists():
|
||||
saved = json.loads(config.read_text())
|
||||
if saved.get('controller', '').rstrip('/') != sys.argv[1].rstrip('/') or pathlib.Path(saved.get('root', '')).resolve() != pathlib.Path(sys.argv[2]).resolve():
|
||||
python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY'
|
||||
# INSTALLATION_PREFLIGHT: this runs before touching managed directories or services.
|
||||
import json, pathlib, re, stat, sys
|
||||
|
||||
controller, raw_root, config_name = sys.argv[1:]
|
||||
root = pathlib.Path(raw_root)
|
||||
config = pathlib.Path(config_name)
|
||||
reserved = {
|
||||
'/', '/bin', '/boot', '/dev', '/etc', '/home', '/lib', '/lib64',
|
||||
'/media', '/mnt', '/opt', '/proc', '/root', '/run', '/sbin', '/srv',
|
||||
'/sys', '/tmp', '/usr', '/usr/local', '/usr/local/bin', '/usr/local/lib',
|
||||
'/usr/local/libexec', '/var', '/var/cache', '/var/lib', '/var/log', '/var/tmp',
|
||||
}
|
||||
if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root)
|
||||
or str(root) != raw_root or any(part in ('.', '..') for part in raw_root.split('/')[1:])
|
||||
or len(root.parts) < 3 or raw_root in reserved or config.parent == root or config.parent in root.parents):
|
||||
raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.')
|
||||
installed_paths = [config,
|
||||
pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'),
|
||||
pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'),
|
||||
pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'),
|
||||
pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service')]
|
||||
if any(root == path or root in path.parents for path in installed_paths):
|
||||
raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.')
|
||||
|
||||
def trusted_directories(path):
|
||||
for current in [*reversed(path.parents), path]:
|
||||
try:
|
||||
info = current.lstat()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
|
||||
raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.')
|
||||
|
||||
def trusted_json(path):
|
||||
info = path.lstat()
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
|
||||
raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.')
|
||||
value = json.loads(path.read_text())
|
||||
if not isinstance(value, dict):
|
||||
raise SystemExit('Existing installation metadata is invalid.')
|
||||
return value
|
||||
|
||||
trusted_directories(root)
|
||||
trusted_directories(config.parent)
|
||||
mountinfo = pathlib.Path('/proc/1/mountinfo')
|
||||
if mountinfo.exists():
|
||||
for line in mountinfo.read_text().splitlines():
|
||||
fields = line.split()
|
||||
if len(fields) < 5:
|
||||
continue
|
||||
mount = re.sub(r'\\([0-7]{3})', lambda match: chr(int(match[1], 8)), fields[4])
|
||||
if mount == raw_root or mount.startswith(raw_root + '/'):
|
||||
raise SystemExit('Use a dedicated subdirectory of the mounted disk, with no filesystems mounted inside it, so host deletion can remove the entire data directory.')
|
||||
request = config.with_name('decommission.json')
|
||||
if request.exists() or request.is_symlink():
|
||||
raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.')
|
||||
saved = None
|
||||
if config.exists() or config.is_symlink():
|
||||
saved = trusted_json(config)
|
||||
if saved.get('controller', '').rstrip('/') != controller.rstrip('/') or saved.get('root') != raw_root:
|
||||
raise SystemExit('Existing enrollment uses a different controller or data directory. Preserve the existing installation and migrate it explicitly.')
|
||||
if not saved.get('hostId') or not saved.get('credential'):
|
||||
raise SystemExit('Existing enrollment is incomplete. Repair its credentials before upgrading.')
|
||||
marker = root / '.electric-cherry.json'
|
||||
if marker.exists() or marker.is_symlink():
|
||||
owned = trusted_json(marker)
|
||||
if not saved or owned != {'version': 1, 'root': raw_root, 'hostId': saved['hostId'], 'controller': controller.rstrip('/')}:
|
||||
raise SystemExit('Data directory ownership does not match this enrollment. Preserve it and migrate explicitly.')
|
||||
elif root.exists() and any(root.iterdir()):
|
||||
# Upgrade installations predating ownership markers only when the entire
|
||||
# root has the known layout. Never adopt a directory of unrelated files.
|
||||
if not saved or any(path.name not in {'agent', 'instances', 'libraries'} for path in root.iterdir()):
|
||||
raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.')
|
||||
for directory in ('agent', 'instances', 'libraries'):
|
||||
trusted_directories(root / directory)
|
||||
PY
|
||||
if ! command -v docker >/dev/null; then
|
||||
if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then
|
||||
@@ -76,9 +145,36 @@ install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-
|
||||
install -d -m 0755 /usr/local/libexec
|
||||
install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py
|
||||
}
|
||||
# A separate worker survives stopping/removing the agent service. Its only
|
||||
# entry point is a fixed, authenticated deletion request prepared by the agent.
|
||||
install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py
|
||||
printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token"
|
||||
unset enrollment_token
|
||||
python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"}
|
||||
python3 - "$data_dir" /etc/electric-cherry/agent.json <<'PY'
|
||||
# INSTALLATION_OWNERSHIP: bind deletion authority to this exact enrollment/root.
|
||||
import json, os, pathlib, sys, uuid
|
||||
root = pathlib.Path(sys.argv[1])
|
||||
config = json.loads(pathlib.Path(sys.argv[2]).read_text())
|
||||
if config.get('root') != str(root) or not config.get('hostId') or not config.get('controller'):
|
||||
raise SystemExit('Enrollment did not establish ownership of this data directory.')
|
||||
marker = root / '.electric-cherry.json'
|
||||
temporary = root / ('.electric-cherry.' + uuid.uuid4().hex + '.tmp')
|
||||
try:
|
||||
with temporary.open('x') as stream:
|
||||
os.chmod(temporary, 0o600)
|
||||
json.dump({'version': 1, 'root': str(root), 'hostId': config['hostId'], 'controller': config['controller'].rstrip('/')}, stream)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, marker)
|
||||
directory_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
PY
|
||||
cat > /etc/systemd/system/electric-cherry-agent.service <<EOF
|
||||
[Unit]
|
||||
Description=Electric Cherry Host Agent
|
||||
@@ -99,6 +195,23 @@ ProtectHome=true
|
||||
PrivateTmp=true
|
||||
ReadWritePaths=$data_dir /etc/electric-cherry
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
cat > /etc/systemd/system/electric-cherry-uninstall.service <<'EOF'
|
||||
[Unit]
|
||||
Description=Remove an Electric Cherry Host Installation
|
||||
After=network-online.target docker.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-uninstall.py --uninstall /etc/electric-cherry/decommission.json
|
||||
Restart=on-failure
|
||||
RestartSec=15
|
||||
TimeoutStartSec=0
|
||||
UMask=0077
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
@@ -10,4 +10,4 @@ target = source.parent / "static" / "agent"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(source / "ec_agent.py", target / "ec-agent.py")
|
||||
shutil.copyfile(source / "install.sh", target / "install.sh")
|
||||
(target / "manifest.json").write_text(json.dumps({"version": "1.0.0", "protocolVersion": 1, "platform": "debian-amd64", "sha256": hashlib.sha256((target / "ec-agent.py").read_bytes()).hexdigest()}, indent=2) + "\n")
|
||||
(target / "manifest.json").write_text(json.dumps({"version": "1.1.0", "protocolVersion": 1, "platform": "debian-amd64", "sha256": hashlib.sha256((target / "ec-agent.py").read_bytes()).hexdigest()}, indent=2) + "\n")
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Exercise installer guards without installing software or touching services."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
SOURCE = (Path(__file__).resolve().parents[1] / "install.sh").read_text()
|
||||
BLOCKS = re.findall(r"<<'PY'\n(.*?)\nPY", SOURCE, re.DOTALL)
|
||||
PREFLIGHT = next(block for block in BLOCKS if "# INSTALLATION_PREFLIGHT:" in block)
|
||||
OWNERSHIP = next(block for block in BLOCKS if "# INSTALLATION_OWNERSHIP:" in block)
|
||||
|
||||
|
||||
class InstallerTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.base = Path(self.temporary.name).resolve()
|
||||
self.root = self.base / "managed"
|
||||
self.config = self.base / "configuration" / "agent.json"
|
||||
self.config.parent.mkdir()
|
||||
self.controller = "https://controller.example"
|
||||
self.saved = {"root": str(self.root), "controller": self.controller,
|
||||
"hostId": "host-one", "credential": "private"}
|
||||
self.overrides = {}
|
||||
|
||||
def tearDown(self):
|
||||
self.temporary.cleanup()
|
||||
|
||||
def preflight(self, root=None):
|
||||
original_lstat = Path.lstat
|
||||
|
||||
def root_owned(path):
|
||||
# Simulate Linux root-owned parents while keeping the actual inode
|
||||
# types. No filesystem mutations or elevated privileges are needed.
|
||||
info = original_lstat(path)
|
||||
fields = list(info)
|
||||
fields[4] = 0
|
||||
if stat.S_ISDIR(info.st_mode):
|
||||
fields[0] &= ~0o022
|
||||
override = self.overrides.get(str(path), {})
|
||||
fields[0] = override.get("mode", fields[0])
|
||||
fields[4] = override.get("uid", fields[4])
|
||||
return os.stat_result(fields)
|
||||
|
||||
argv = ["-", self.controller, str(root or self.root), str(self.config)]
|
||||
with patch.object(sys, "argv", argv), patch.object(Path, "lstat", root_owned):
|
||||
exec(compile(PREFLIGHT, "installer-preflight", "exec"), {})
|
||||
|
||||
def legacy_installation(self):
|
||||
for directory in ("agent", "instances", "libraries"):
|
||||
(self.root / directory).mkdir(parents=True, exist_ok=True)
|
||||
self.config.write_text(json.dumps(self.saved))
|
||||
|
||||
def ownership(self):
|
||||
with patch.object(sys, "argv", ["-", str(self.root), str(self.config)]):
|
||||
exec(compile(OWNERSHIP, "installer-ownership", "exec"), {})
|
||||
|
||||
def test_new_directory_and_empty_directory_are_allowed_without_adopting_files(self):
|
||||
self.preflight()
|
||||
self.assertFalse(self.root.exists())
|
||||
self.root.mkdir()
|
||||
self.preflight()
|
||||
(self.root / "unrelated.txt").write_text("keep")
|
||||
with self.assertRaisesRegex(SystemExit, "not owned"):
|
||||
self.preflight()
|
||||
self.assertEqual((self.root / "unrelated.txt").read_text(), "keep")
|
||||
|
||||
def test_known_legacy_layout_can_upgrade_and_marker_binds_exact_enrollment(self):
|
||||
self.legacy_installation()
|
||||
original_config = self.config.read_bytes()
|
||||
(self.root / "libraries" / "game.ff").write_bytes(b"game data")
|
||||
self.preflight()
|
||||
self.ownership()
|
||||
marker = self.root / ".electric-cherry.json"
|
||||
self.assertEqual(json.loads(marker.read_text()), {
|
||||
"version": 1, "root": str(self.root), "hostId": "host-one", "controller": self.controller})
|
||||
self.assertEqual(stat.S_IMODE(marker.stat().st_mode), 0o600)
|
||||
self.assertEqual(self.config.read_bytes(), original_config)
|
||||
self.assertEqual((self.root / "libraries" / "game.ff").read_bytes(), b"game data")
|
||||
self.preflight()
|
||||
|
||||
def test_legacy_upgrade_never_adopts_unrelated_top_level_files(self):
|
||||
self.legacy_installation()
|
||||
(self.root / "personal-backup.tar").write_bytes(b"keep")
|
||||
with self.assertRaisesRegex(SystemExit, "not owned"):
|
||||
self.preflight()
|
||||
self.assertFalse((self.root / ".electric-cherry.json").exists())
|
||||
|
||||
def test_reenrollment_cannot_silently_change_controller_or_root(self):
|
||||
self.legacy_installation()
|
||||
for field, value in (("controller", "https://another.example"), ("root", str(self.base / "other"))):
|
||||
with self.subTest(field=field):
|
||||
self.config.write_text(json.dumps({**self.saved, field: value}))
|
||||
with self.assertRaisesRegex(SystemExit, "different controller or data directory"):
|
||||
self.preflight()
|
||||
|
||||
def test_existing_marker_must_match_controller_host_and_root(self):
|
||||
self.legacy_installation()
|
||||
self.ownership()
|
||||
marker = self.root / ".electric-cherry.json"
|
||||
valid = json.loads(marker.read_text())
|
||||
for field, value in (("hostId", "host-two"), ("root", str(self.base)), ("controller", "https://another.example")):
|
||||
with self.subTest(field=field):
|
||||
marker.write_text(json.dumps({**valid, field: value}))
|
||||
with self.assertRaisesRegex(SystemExit, "ownership does not match"):
|
||||
self.preflight()
|
||||
|
||||
def test_lexical_aliases_and_system_roots_are_rejected_before_mutation(self):
|
||||
for root in ("/var/lib/..", "/var/lib/../lib", "/var//lib/managed", "/var/lib/./managed", "/var/lib/managed/", "/var/lib", "/usr/local/libexec", "/", "/games", str(self.config.parent), str(self.config.parent / "data")):
|
||||
with self.subTest(root=root), self.assertRaisesRegex(SystemExit, "canonical, dedicated"):
|
||||
self.preflight(root)
|
||||
self.assertFalse(self.root.exists())
|
||||
|
||||
def test_symlink_roots_ancestors_metadata_and_managed_subdirectories_are_rejected(self):
|
||||
elsewhere = self.base / "elsewhere"
|
||||
elsewhere.mkdir()
|
||||
self.root.symlink_to(elsewhere, target_is_directory=True)
|
||||
with self.assertRaisesRegex(SystemExit, "not symlinks"):
|
||||
self.preflight()
|
||||
with self.assertRaisesRegex(SystemExit, "not symlinks"):
|
||||
self.preflight(str(self.root / "nested"))
|
||||
self.root.unlink()
|
||||
self.legacy_installation()
|
||||
(self.root / "libraries").rmdir()
|
||||
(self.root / "libraries").symlink_to(elsewhere, target_is_directory=True)
|
||||
with self.assertRaisesRegex(SystemExit, "not symlinks"):
|
||||
self.preflight()
|
||||
(self.root / "libraries").unlink()
|
||||
(self.root / "libraries").mkdir()
|
||||
self.config.unlink()
|
||||
real_config = self.base / "real-config.json"
|
||||
real_config.write_text(json.dumps(self.saved))
|
||||
self.config.symlink_to(real_config)
|
||||
with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
|
||||
self.preflight()
|
||||
|
||||
def test_untrusted_parents_and_metadata_cannot_authorize_cleanup(self):
|
||||
self.legacy_installation()
|
||||
self.overrides[str(self.base)] = {"uid": 1000}
|
||||
with self.assertRaisesRegex(SystemExit, "root-owned"):
|
||||
self.preflight()
|
||||
self.overrides = {str(self.base): {"mode": stat.S_IFDIR | 0o777}}
|
||||
with self.assertRaisesRegex(SystemExit, "root-owned"):
|
||||
self.preflight()
|
||||
self.overrides = {str(self.config): {"uid": 1000}}
|
||||
with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
|
||||
self.preflight()
|
||||
|
||||
def test_pending_deletion_blocks_upgrade_even_with_dangling_request_symlink(self):
|
||||
self.legacy_installation()
|
||||
request = self.config.with_name("decommission.json")
|
||||
request.write_text("{}")
|
||||
with self.assertRaisesRegex(SystemExit, "deletion is in progress"):
|
||||
self.preflight()
|
||||
request.unlink()
|
||||
request.symlink_to(self.base / "missing")
|
||||
with self.assertRaisesRegex(SystemExit, "deletion is in progress"):
|
||||
self.preflight()
|
||||
|
||||
def test_mounted_parent_is_allowed_but_data_root_and_nested_mounts_are_rejected(self):
|
||||
mountinfo = Path('/proc/1/mountinfo')
|
||||
original_exists, original_read = Path.exists, Path.read_text
|
||||
for mount, allowed in ((self.base, True), (self.root, False), (self.root / 'libraries', False)):
|
||||
def exists(path):
|
||||
return True if path == mountinfo else original_exists(path)
|
||||
|
||||
def read(path, *args, **kwargs):
|
||||
return f'29 24 8:1 / {mount} rw - ext4 /dev/sda1 rw' if path == mountinfo else original_read(path, *args, **kwargs)
|
||||
|
||||
with self.subTest(mount=mount), patch.object(Path, 'exists', exists), patch.object(Path, 'read_text', read):
|
||||
if allowed:
|
||||
self.preflight()
|
||||
else:
|
||||
with self.assertRaisesRegex(SystemExit, "dedicated subdirectory of the mounted disk"):
|
||||
self.preflight()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,338 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location("ec_uninstall_agent", Path(__file__).resolve().parents[1] / "ec_agent.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
INSTANCE = "a53d99da-a1eb-4f01-9b10-a6268b556542"
|
||||
|
||||
|
||||
class CleanupDocker:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
self.items = {}
|
||||
self.images = {}
|
||||
|
||||
def inspect(self, identity):
|
||||
return self.items.get(identity)
|
||||
|
||||
def call(self, *args, **kwargs):
|
||||
self.calls.append(args)
|
||||
if args[:2] == ("container", "ls"):
|
||||
return "\n".join(self.items)
|
||||
if args[:2] == ("container", "rm"):
|
||||
self.items.pop(args[-1], None)
|
||||
return ""
|
||||
if args[:2] == ("image", "inspect"):
|
||||
item = self.images.get(args[-1])
|
||||
return json.dumps([item]) if item else None
|
||||
if args[:2] == ("image", "rm"):
|
||||
self.images.pop(args[-1], None)
|
||||
return ""
|
||||
raise AssertionError("Unexpected Docker request: " + str(args))
|
||||
|
||||
|
||||
class UninstallTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.base = Path(self.temp.name).resolve()
|
||||
self.root = self.base / "host-data"
|
||||
self.config_dir = self.base / "etc" / "electric-cherry"
|
||||
self.bin_dir = self.base / "libexec"
|
||||
self.unit_dir = self.base / "systemd"
|
||||
for directory in (self.config_dir, self.bin_dir, self.unit_dir):
|
||||
directory.mkdir(parents=True)
|
||||
replacements = {
|
||||
"AGENT_CONFIG": self.config_dir / "agent.json",
|
||||
"AGENT_EXECUTABLE": self.bin_dir / "electric-cherry-agent.py",
|
||||
"AGENT_UNIT": self.unit_dir / "electric-cherry-agent.service",
|
||||
"UNINSTALL_EXECUTABLE": self.bin_dir / "electric-cherry-uninstall.py",
|
||||
"UNINSTALL_UNIT": self.unit_dir / "electric-cherry-uninstall.service",
|
||||
"UNINSTALL_REQUEST": self.config_dir / "decommission.json",
|
||||
}
|
||||
self.patches = [patch.object(module, key, value) for key, value in replacements.items()]
|
||||
# Unit tests run without root privileges. Keep actual path, symlink,
|
||||
# file-type and containment checks while substituting owner checks.
|
||||
def test_protected_path(value, missing=False):
|
||||
path = Path(value)
|
||||
if any(current.is_symlink() for current in (path, *path.parents)):
|
||||
raise module.AgentError("Symlink ownership is invalid")
|
||||
if not missing and not path.exists():
|
||||
raise module.AgentError("Host uninstall files are unavailable")
|
||||
return path
|
||||
self.patches.extend([patch.object(module, "protected_path", side_effect=test_protected_path), patch.object(module, "systemctl"), patch.object(module, "sync_parent")])
|
||||
for entry in self.patches:
|
||||
entry.start()
|
||||
self.config = {"hostId": "host-one", "controller": "https://controller.example", "credential": "secret", "root": str(self.root), "configPath": str(module.AGENT_CONFIG)}
|
||||
self.agent = module.Agent(self.config)
|
||||
self.docker = CleanupDocker()
|
||||
for path in (module.AGENT_EXECUTABLE, module.AGENT_UNIT, module.UNINSTALL_EXECUTABLE, module.UNINSTALL_UNIT):
|
||||
path.write_text("installed")
|
||||
module.atomic_json(module.AGENT_CONFIG, self.config)
|
||||
module.atomic_json(self.root / module.ROOT_MARKER, {"version": 1, "root": str(self.root), "hostId": "host-one", "controller": "https://controller.example"})
|
||||
self.game = self.root / "libraries" / "t6" / "library" / "content" / "zone" / "all" / "dlc.ff"
|
||||
self.game.parent.mkdir(parents=True)
|
||||
self.game.write_bytes(b"game DLC")
|
||||
(self.base / "unrelated.txt").write_text("keep")
|
||||
self.operation = {"id": "delete-one", "kind": "host.decommission", "payload": {"hostId": "host-one", "operationId": "delete-one", "receiptToken": "receipt-only-secret", "receiptUrl": "/api/agent/decommission/delete-one"}}
|
||||
|
||||
def tearDown(self):
|
||||
self.agent.journal.close()
|
||||
for entry in reversed(self.patches):
|
||||
entry.stop()
|
||||
self.temp.cleanup()
|
||||
|
||||
def prepare(self):
|
||||
result = self.agent.execute(self.operation)
|
||||
self.assertEqual(result["state"], "running", result)
|
||||
return module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
|
||||
|
||||
def container(self, identity, owned=True, host="host-one", root=None, image="image-one"):
|
||||
labels = {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1", module.LABEL + ".host": host} if owned else {}
|
||||
return {"Id": identity, "Image": image, "Config": {"Labels": labels}, "State": {"Running": False, "Status": "exited"}, "Mounts": [{"Type": "bind", "Source": str((root or self.root) / "instances" / INSTANCE / "1" / "app"), "Destination": "/home/plutainer/app"}]}
|
||||
|
||||
def test_final_receipt_follows_data_and_self_removal(self):
|
||||
worker = self.prepare()
|
||||
stages = []
|
||||
def receipt(stage, error=None):
|
||||
stages.append(stage)
|
||||
self.assertFalse(self.root.exists())
|
||||
self.assertFalse(module.AGENT_CONFIG.exists())
|
||||
self.assertFalse(module.AGENT_EXECUTABLE.exists())
|
||||
self.assertFalse(module.AGENT_UNIT.exists())
|
||||
if stage == "cleaned":
|
||||
self.assertTrue(module.UNINSTALL_REQUEST.exists())
|
||||
self.assertTrue(module.UNINSTALL_EXECUTABLE.exists())
|
||||
else:
|
||||
self.assertFalse(self.config_dir.exists())
|
||||
self.assertFalse(module.UNINSTALL_EXECUTABLE.exists())
|
||||
self.assertFalse(module.UNINSTALL_UNIT.exists())
|
||||
worker.receipt = receipt
|
||||
worker.run()
|
||||
self.assertEqual(stages, ["cleaned", "completed"])
|
||||
self.assertEqual((self.base / "unrelated.txt").read_text(), "keep")
|
||||
module.systemctl.assert_any_call("disable", "--now", "electric-cherry-agent.service")
|
||||
module.systemctl.assert_any_call("disable", "electric-cherry-uninstall.service")
|
||||
|
||||
def test_handoff_is_durable_and_never_reports_operation_completed(self):
|
||||
worker = self.prepare()
|
||||
self.assertEqual(worker.job["operationId"], "delete-one")
|
||||
self.assertNotIn("credential", worker.job)
|
||||
self.assertTrue(self.agent.stopping.is_set())
|
||||
module.systemctl.assert_any_call("enable", "--now", "--no-block", "electric-cherry-uninstall.service")
|
||||
self.assertEqual(self.agent.journal.get("delete-one")["state"], "running")
|
||||
|
||||
def test_failed_checkpoint_cannot_leave_a_runnable_uninstall_request(self):
|
||||
with patch.object(self.agent.journal, "checkpoint", side_effect=OSError("journal unavailable")):
|
||||
result = self.agent.execute(self.operation)
|
||||
self.assertEqual(result["state"], "failed")
|
||||
self.assertFalse(module.UNINSTALL_REQUEST.exists())
|
||||
module.systemctl.assert_not_called()
|
||||
|
||||
def test_pending_cleanup_resumes_after_controller_unavailable(self):
|
||||
worker = self.prepare()
|
||||
worker.receipt = lambda *_: (_ for _ in ()).throw(module.AgentError("offline"))
|
||||
with self.assertRaisesRegex(module.AgentError, "offline"):
|
||||
worker.run()
|
||||
self.assertFalse(self.root.exists())
|
||||
self.assertTrue(module.UNINSTALL_REQUEST.exists())
|
||||
self.assertTrue(module.UNINSTALL_EXECUTABLE.exists())
|
||||
self.assertTrue(json.loads(module.UNINSTALL_REQUEST.read_text())["dataRemoved"])
|
||||
before = len(self.docker.calls)
|
||||
resumed = module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
|
||||
stages = []
|
||||
resumed.receipt = lambda stage, error=None: stages.append(stage)
|
||||
resumed.run()
|
||||
self.assertEqual(stages, ["cleaned", "completed"])
|
||||
self.assertEqual(len(self.docker.calls), before)
|
||||
|
||||
def test_lost_final_receipt_retries_after_all_files_are_gone(self):
|
||||
worker = self.prepare()
|
||||
stages = []
|
||||
def receipt(stage, error=None):
|
||||
stages.append(stage)
|
||||
if stage == "completed":
|
||||
self.assertFalse(self.config_dir.exists())
|
||||
if stages.count("completed") == 1:
|
||||
raise module.AgentError("lost reply")
|
||||
worker.receipt = receipt
|
||||
with patch.object(module.time, "sleep") as wait:
|
||||
worker.run()
|
||||
self.assertEqual(stages, ["cleaned", "completed", "completed"])
|
||||
wait.assert_called_once_with(15)
|
||||
|
||||
def test_partial_self_removal_retries_without_its_unit_or_executable(self):
|
||||
worker = self.prepare()
|
||||
stages = []
|
||||
worker.receipt = lambda stage, error=None: stages.append(stage)
|
||||
original = shutil.rmtree
|
||||
attempts = 0
|
||||
def interrupted(path, *args, **kwargs):
|
||||
nonlocal attempts
|
||||
if Path(path) == self.config_dir:
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
self.assertFalse(module.UNINSTALL_UNIT.exists())
|
||||
self.assertFalse(module.UNINSTALL_EXECUTABLE.exists())
|
||||
raise OSError("temporarily busy")
|
||||
return original(path, *args, **kwargs)
|
||||
with patch.object(module.shutil, "rmtree", side_effect=interrupted), patch.object(module.time, "sleep") as wait:
|
||||
worker.run()
|
||||
self.assertEqual(stages, ["cleaned", "failed", "completed"])
|
||||
self.assertFalse(self.config_dir.exists())
|
||||
wait.assert_called_once_with(15)
|
||||
|
||||
def test_partial_tree_removal_recovers_without_deleted_marker(self):
|
||||
worker = self.prepare()
|
||||
original = shutil.rmtree
|
||||
def interrupted(path, *args, **kwargs):
|
||||
if Path(path) == self.root:
|
||||
(self.root / module.ROOT_MARKER).unlink()
|
||||
self.game.unlink()
|
||||
raise OSError("power loss")
|
||||
return original(path, *args, **kwargs)
|
||||
worker.receipt = lambda *_: None
|
||||
with patch.object(module.shutil, "rmtree", side_effect=interrupted):
|
||||
with self.assertRaisesRegex(OSError, "power loss"):
|
||||
worker.run()
|
||||
self.assertTrue(json.loads(module.UNINSTALL_REQUEST.read_text())["removalStarted"])
|
||||
self.assertFalse((self.root / module.ROOT_MARKER).exists())
|
||||
resumed = module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
|
||||
resumed.receipt = lambda *_: None
|
||||
resumed.run()
|
||||
self.assertFalse(self.root.exists())
|
||||
|
||||
def test_replaced_data_root_cannot_be_deleted_on_resume(self):
|
||||
self.prepare()
|
||||
self.root.rename(self.base / "original-data")
|
||||
self.root.mkdir()
|
||||
(self.root / "precious.txt").write_text("keep")
|
||||
with self.assertRaisesRegex(module.AgentError, "changed"):
|
||||
module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
|
||||
self.assertTrue((self.root / "precious.txt").exists())
|
||||
|
||||
def test_only_owned_containers_and_unused_recorded_images_are_removed(self):
|
||||
aliases = {"example/runtime:one": "image-one", "example/runtime:shared": "image-shared", "example/runtime:changed": "image-old"}
|
||||
module.atomic_json(self.root / "agent" / "runtime-images.json", aliases)
|
||||
self.docker.images = {"example/runtime:one": {"Id": "image-one"}, "example/runtime:shared": {"Id": "image-shared"}, "example/runtime:changed": {"Id": "image-new"}, "example/unrelated:one": {"Id": "image-unrelated"}}
|
||||
self.docker.items = {"ours": self.container("ours"), "other": self.container("other", host="host-two", root=self.base / "different-install", image="image-shared")}
|
||||
worker = self.prepare()
|
||||
worker.receipt = lambda *_: None
|
||||
worker.run()
|
||||
self.assertNotIn("ours", self.docker.items)
|
||||
self.assertIn("other", self.docker.items)
|
||||
self.assertNotIn("example/runtime:one", self.docker.images)
|
||||
self.assertIn("example/runtime:shared", self.docker.images)
|
||||
self.assertIn("example/runtime:changed", self.docker.images)
|
||||
self.assertIn("example/unrelated:one", self.docker.images)
|
||||
self.assertFalse(any("prune" in call or "volume" in call for call in self.docker.calls))
|
||||
|
||||
def test_unrelated_container_using_root_blocks_any_removal(self):
|
||||
self.docker.items = {"foreign": self.container("foreign", owned=False)}
|
||||
worker = self.prepare()
|
||||
stages = []
|
||||
worker.receipt = lambda stage, error=None: stages.append((stage, error))
|
||||
with self.assertRaisesRegex(module.AgentError, "unrelated container"):
|
||||
worker.run()
|
||||
self.assertEqual(stages[0][0], "failed")
|
||||
self.assertTrue(self.game.exists())
|
||||
self.assertTrue(module.AGENT_EXECUTABLE.exists())
|
||||
self.assertIn("foreign", self.docker.items)
|
||||
|
||||
def test_running_or_unknown_managed_container_blocks_removal(self):
|
||||
worker = self.prepare()
|
||||
worker.receipt = lambda *_: None
|
||||
for state in ({"Running": True, "Status": "running"}, {"Running": False, "Status": "restarting"}, {}):
|
||||
with self.subTest(state=state):
|
||||
self.docker.items = {"ours": {**self.container("ours"), "State": state}}
|
||||
with self.assertRaisesRegex(module.AgentError, "still running or its state is unknown"):
|
||||
worker.run()
|
||||
self.assertTrue(self.game.exists())
|
||||
self.assertIn("ours", self.docker.items)
|
||||
self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
|
||||
|
||||
def test_legacy_container_requires_exact_managed_bind_mount(self):
|
||||
worker = self.prepare()
|
||||
legacy = self.container("legacy")
|
||||
legacy["Config"]["Labels"].pop(module.LABEL + ".host")
|
||||
self.assertTrue(worker.owned_container(legacy))
|
||||
legacy["Mounts"][0]["Source"] = str(self.base / "elsewhere")
|
||||
self.assertFalse(worker.owned_container(legacy))
|
||||
|
||||
def test_missing_marker_and_wrong_host_never_schedule_helper(self):
|
||||
(self.root / module.ROOT_MARKER).unlink()
|
||||
self.assertFalse(self.agent.can_uninstall())
|
||||
result = self.agent.execute(self.operation)
|
||||
self.assertEqual(result["state"], "failed")
|
||||
module.systemctl.assert_not_called()
|
||||
self.assertTrue(self.game.exists())
|
||||
|
||||
def test_receipt_path_and_identity_cannot_be_redirected(self):
|
||||
for key, value in (("receiptUrl", "https://attacker.example"), ("hostId", "other"), ("operationId", "other")):
|
||||
with self.subTest(key=key):
|
||||
operation = {**self.operation, "id": "check-" + key, "payload": {**self.operation["payload"], "operationId": "check-" + key, "receiptUrl": "/api/agent/decommission/check-" + key, key: value}}
|
||||
self.assertEqual(self.agent.execute(operation)["state"], "failed")
|
||||
module.systemctl.assert_not_called()
|
||||
|
||||
def test_symlink_inside_owned_tree_is_unlinked_without_following(self):
|
||||
outside = self.base / "unrelated.txt"
|
||||
(self.root / "libraries" / "symlink").symlink_to(outside)
|
||||
worker = self.prepare()
|
||||
worker.receipt = lambda *_: None
|
||||
worker.run()
|
||||
self.assertEqual(outside.read_text(), "keep")
|
||||
|
||||
def test_unsafe_roots_and_installation_overlap_are_rejected(self):
|
||||
for value in ("/", "/var/lib", "/srv", "/var/lib/../lib", "/var//lib/anything", str(self.config_dir), str(self.bin_dir)):
|
||||
with self.subTest(value=value):
|
||||
with self.assertRaises(module.AgentError):
|
||||
module.uninstall_root(value)
|
||||
|
||||
def test_root_mount_or_nested_mount_cannot_be_recursively_deleted(self):
|
||||
original = Path.read_text
|
||||
for mount in (self.root, self.root / "libraries"):
|
||||
def read_text(path, *args, **kwargs):
|
||||
if str(path) == "/proc/1/mountinfo":
|
||||
return "21 1 8:1 / " + str(mount) + " rw - ext4 /dev/sda1 rw\n"
|
||||
return original(path, *args, **kwargs)
|
||||
with patch.object(Path, "read_text", read_text):
|
||||
with self.assertRaisesRegex(module.AgentError, "Unmount"):
|
||||
module.uninstall_root(str(self.root))
|
||||
|
||||
def test_marker_directory_is_rejected_before_reading(self):
|
||||
marker = self.root / module.ROOT_MARKER
|
||||
marker.unlink()
|
||||
marker.mkdir()
|
||||
with self.assertRaisesRegex(module.AgentError, "regular file"):
|
||||
module.uninstall_ownership(self.config)
|
||||
|
||||
def test_boot_reconciles_handoff_instead_of_marking_complete(self):
|
||||
self.prepare()
|
||||
self.agent.journal.close()
|
||||
self.agent = module.Agent(self.config)
|
||||
self.assertEqual(self.agent.journal.get("delete-one")["state"], "unknown")
|
||||
result = self.agent.execute(self.operation)
|
||||
self.assertEqual(result["state"], "running")
|
||||
self.assertEqual(json.loads(module.UNINSTALL_REQUEST.read_text())["operationId"], "delete-one")
|
||||
|
||||
|
||||
class OwnershipPathTests(unittest.TestCase):
|
||||
def test_root_ownership_group_writes_and_symlinks_are_rejected(self):
|
||||
for mode, uid in ((stat.S_IFDIR | 0o777, 0), (stat.S_IFDIR | 0o755, 1000), (stat.S_IFLNK | 0o777, 0)):
|
||||
with self.subTest(mode=mode, uid=uid):
|
||||
value = os.stat_result((mode, 1, 1, 1, uid, 0, 0, 0, 0, 0))
|
||||
with patch.object(Path, "lstat", return_value=value):
|
||||
with self.assertRaisesRegex(module.AgentError, "only by root"):
|
||||
module.protected_path("/var/lib/electric-cherry")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -31,7 +31,7 @@ guessable token. An unset setup token holds setup closed.
|
||||
| Raw configs, file diffs, logs and telemetry diagnostics | No | Yes | Yes |
|
||||
| Deploy/start/stop/restart/rebuild/remove servers | No | Yes | Yes |
|
||||
| Scripts, plugins, configs, campaigns, betas and collections | Read summaries | Manage | Manage |
|
||||
| Host setup, enrollment, credentials, runtime preparation and game-file import/cache | Read status | Read status | Manage |
|
||||
| Host setup, deletion, enrollment, credentials, runtime preparation and game-file upload/cache | Read status | Read status | Manage |
|
||||
| Server keys and global settings/maintenance policies | No | No | Manage |
|
||||
| Create users, change roles, disable accounts and reset passwords | No | No | Yes |
|
||||
| Own password and sign out | Yes | Yes | Yes |
|
||||
@@ -85,6 +85,9 @@ endpoint or accept passwords as command arguments.
|
||||
Host agents retain enrollment/host-token authentication under `/api/agent/`.
|
||||
Installer and release files remain public so a new machine can enroll. A browser
|
||||
login never substitutes for a host credential, and host tokens cannot manage EC.
|
||||
Host deletion uses a separate operation-scoped receipt token so the uninstall
|
||||
worker can confirm removal and retry its acknowledgment after host credentials
|
||||
are revoked. That token authorizes only the corresponding cleanup receipt.
|
||||
|
||||
Game reporters must send a separate secret of at least 32 characters configured
|
||||
as `EC_TELEMETRY_TOKEN`:
|
||||
|
||||
@@ -131,7 +131,7 @@ the typed key so it cannot accidentally be saved against a different lobby.
|
||||
|
||||
Komodo connection settings, target host, RCON configuration, config filename,
|
||||
lobby availability, and port availability are checked locally. These checks do
|
||||
not probe the host or verify game/DLC files, images, Docker volumes, firewall
|
||||
not probe the host or verify game files, images, Docker volumes, firewall
|
||||
rules, or credentials. Container and RCON verification happen after provisioning.
|
||||
T6 variants retain their exact catalog slug and launch ruleset; RCON health checks
|
||||
verify the engine map, not the variant's start location.
|
||||
|
||||
+92
-11
@@ -17,12 +17,16 @@ the agent owns local execution and its durable operation journal.
|
||||
3. The installer validates the platform, installs Python 3 and an absent Docker
|
||||
installation, verifies the downloaded agent SHA-256 over HTTPS, enrolls it and
|
||||
enables `electric-cherry-agent.service`. It needs no Git, Node.js or local build.
|
||||
4. In **Hosts**, prepare a Plutainer runtime image. In **Game Files**, import your
|
||||
existing `t5_game_files` or `t6_game_files` Docker volume on an enrolled source
|
||||
host. Name the library and version and record its DLC metadata.
|
||||
4. In **Hosts**, prepare a Plutainer runtime image. In **Game Files**, select T5 or
|
||||
T6 and upload your local game folder. Name the library and version and publish
|
||||
it after the upload finishes. Include the DLC files in that folder; there is
|
||||
no separate DLC declaration. Uploading also works before enrolling any host.
|
||||
5. Cache the published version on each destination host. Wait for verification.
|
||||
Hosts report connection and readiness separately; deployment also checks the
|
||||
particular library, image, capacity and ports you selected.
|
||||
particular library, image, capacity and ports you selected. A connected host
|
||||
missing content shows **Needs runtime** or **Needs game files**; these require
|
||||
setup rather than waiting for an automatic preparation step. **Ready** means
|
||||
it has both a runtime image and a verified library available.
|
||||
6. Launch a server or collection. Select the host, immutable game-file version(s)
|
||||
and runtime image. EC selects lobby IDs and reserves UDP/TCP ports atomically.
|
||||
Saved Plutonium server keys remain associated with their exact server names.
|
||||
@@ -31,7 +35,10 @@ A fresh destination host downloads centrally stored game files without manual
|
||||
copying or per-container setup. EC does not supply the game content itself.
|
||||
|
||||
The default host root is `/var/lib/electric-cherry`; installer `--data-dir` changes
|
||||
it. Data is separated into `libraries/<game>/<version>/content`,
|
||||
it. Use a dedicated, root-owned directory. For a separate disk, use a child of its
|
||||
mount point (for example `/mnt/games/electric-cherry`), with no nested mounts, so
|
||||
host deletion can remove the entire directory. The installer records ownership
|
||||
and refuses to adopt unrelated files. Data is separated into `libraries/<game>/<version>/content`,
|
||||
`instances/<UUID>/<generation>/app`, and `agent/operations.sqlite3`. Base libraries
|
||||
are mounted read-only and instance deletion removes only that instance's files.
|
||||
The controller stores large files under `GAME_FILES_ROOT` (default beside the
|
||||
@@ -69,12 +76,25 @@ explicit installer `--reenroll` when local credentials already exist.
|
||||
|
||||
API outline:
|
||||
|
||||
- `GET /api/hosts/:id`: host details, operation history and deletion blockers.
|
||||
- `DELETE /api/hosts/:id`: administrator requests cleanup of an unused host;
|
||||
returns `202` while remote removal is pending, `200` for an unenrolled host,
|
||||
or `409` when the agent needs updating or unresolved ownership remains.
|
||||
- `POST /api/agent/enroll`: consume enrollment token and return host credentials.
|
||||
- `POST /api/agent/heartbeat`: authenticated inventory, current operation and caches.
|
||||
- `GET /api/agent/operations`: deliver/reconcile durable operation IDs.
|
||||
- `POST /api/agent/operations/:id`: authenticated acceptance, progress or result.
|
||||
- `/api/agent/imports/...` and `/api/agent/libraries/...`: host-scoped uploads and
|
||||
authenticated game-file downloads.
|
||||
- `POST /api/agent/decommission/:id`: cleanup worker reports file removal and
|
||||
agent uninstall using its own operation-scoped receipt credential.
|
||||
- `POST /api/game-files`: administrator creates a folder upload with game, name,
|
||||
version, optional description and `files: [{path, size, lastModified}]`.
|
||||
- `GET /api/game-files/:id/upload`: durable per-file offsets and publication status.
|
||||
- `PUT /api/game-files/:id/upload/files/:path`: administrator uploads a raw chunk
|
||||
with `X-Upload-Offset`, `X-Upload-Length` and `X-Content-SHA256` headers.
|
||||
- `POST /api/game-files/:id/upload/complete`: administrator starts asynchronous
|
||||
verification/publication; poll the upload endpoint for completion.
|
||||
- `/api/agent/libraries/...`: authenticated game-file downloads. Legacy
|
||||
`/api/agent/imports/...` endpoints remain compatible with already queued imports.
|
||||
|
||||
The operation envelope is independent of polling and can be carried over a
|
||||
persistent transport later. No unrestricted shell operation is available.
|
||||
@@ -115,17 +135,72 @@ may still join until an eligible empty moment; draining does not kick them.
|
||||
**Disable** holds agent dispatch. Neither mode, controller downtime nor agent
|
||||
communication loss stops unrelated running containers.
|
||||
|
||||
Administrators can choose **Delete host** on a host card or at the top of its
|
||||
detail page. The confirmation lists anything that must be resolved first:
|
||||
registered servers, reserved ports, or operations already sent to the agent.
|
||||
Drain the host or remove its servers, then let outstanding operations reconcile.
|
||||
An unenrolled host is removed immediately. An enrolled host stays visible as
|
||||
**Deleting** until its agent confirms cleanup. Offline hosts wait for their next
|
||||
connection; their absence is never treated as proof of removal. Existing agents
|
||||
need the 1.1.0 installer: generate a fresh command under **Agent access**, rerun it
|
||||
on the host, and wait for a heartbeat advertising uninstall support.
|
||||
|
||||
Deletion blocks new operations, cancels queued commands that were never delivered,
|
||||
and invalidates unused installer links. A separate cleanup service removes the
|
||||
host's downloaded game libraries, instance files, agent journal and configuration,
|
||||
owned containers, and recorded runtime images unused by other containers. It
|
||||
stops and removes the agent service and executable, then removes its own files.
|
||||
The controller hides the host and revokes its ordinary credentials only after
|
||||
receiving the final uninstall receipt. Adding the machine again requires a new
|
||||
host enrollment.
|
||||
|
||||
Game folders uploaded through Electric Cherry belong to the controller. Those
|
||||
original uploads remain available for other hosts and future deployments; only
|
||||
the deleted host's copies are removed. Controller operation history is also kept.
|
||||
Unrelated containers, files and shared dependencies such as Docker and Python
|
||||
are preserved. Historical entries in shared system logs are not erased.
|
||||
|
||||
Cleanup retries after connection loss or partial file removal. Failures before
|
||||
handoff offer **Retry cleanup** after the installation is repaired; failures in
|
||||
the independent worker retry automatically. The worker stays installed until
|
||||
the controller acknowledges data cleanup. After removing its final files it
|
||||
retries the completion receipt from memory. A power loss in that final window
|
||||
can leave the controller awaiting confirmation even though removal succeeded;
|
||||
inspect the machine if that status persists. The controller does not guess.
|
||||
|
||||
## Libraries and ports
|
||||
|
||||
Imports scan the source volume and generate per-file SHA-256 manifests. Uploads
|
||||
stream to temporary controller storage, are bounded by their manifest sizes and
|
||||
verified before atomic publication. Published versions cannot be updated;
|
||||
Choose the actual game folder, not its parent, a ZIP archive or the Plutonium
|
||||
launcher data folder. The selected folder's name is removed once; nested paths
|
||||
such as `zone/english/...` and `main/...` stay intact. Plutonium's
|
||||
[T5 server guide](https://plutonium.pw/docs/server/t5/setting-up-a-server/) and
|
||||
[T6 server guide](https://plutonium.pw/docs/server/t6/setting-up-a-server/) use a
|
||||
copy of the game directory, separately from the launcher/runtime. The uploader
|
||||
preserves the supplied files rather than attempting to infer installed maps from
|
||||
a manually maintained DLC list.
|
||||
|
||||
Folder uploads use chunks of at most 8 MiB, each checked against its SHA-256 before
|
||||
being appended to temporary controller storage. Memory usage does not grow with
|
||||
the size of an individual game file. Pause/retry uses the controller's saved file
|
||||
offsets. After a page reload, choose **Resume upload** and reselect the original,
|
||||
unchanged folder; relative paths, sizes and modification times must match. Keep
|
||||
the page open while transferring. Once publication starts, the controller hashes
|
||||
every complete file and publishes the directory atomically; closing the browser
|
||||
does not stop that verification. An interrupted controller can retry publication.
|
||||
Published versions cannot be updated;
|
||||
changing files requires a new version. Hosts download with HTTP Range resumption,
|
||||
report progress, verify all checksums and atomically promote the complete cache.
|
||||
Corrupt or incomplete content never becomes Ready. Cache inventory is reconciled
|
||||
on heartbeats, including vanished caches. Older immutable versions remain usable
|
||||
when their exact manifest is verified.
|
||||
|
||||
The controller and reverse proxy must accept 8 MiB upload requests and folder
|
||||
manifests up to 32 MiB. Compose sets `BODY_SIZE_LIMIT=Infinity`; application routes
|
||||
still enforce these limits, declared file sizes and checksums. Set a proxy body
|
||||
limit of at least 32 MiB (for example `client_max_body_size 32m` in nginx).
|
||||
Existing published versions and cached host libraries remain usable after the
|
||||
upload workflow replaces source-host imports.
|
||||
|
||||
Reservations include host ID, local bind address, protocol and port. Wildcard
|
||||
binds conflict with concrete addresses on the same host. UDP and TCP reservations
|
||||
for one launch commit together; the same number can be used independently on
|
||||
@@ -150,7 +225,9 @@ Automated tests cover enrollment/authentication, rotation/revocation, host
|
||||
readiness, scoped ports, immutable library publication, streaming/ranges,
|
||||
checksums, interrupted downloads, instance generation ownership, local occupancy
|
||||
holds, controller restart/reconnect, ambiguous results, occupied campaign staging,
|
||||
activation before images, collections and draining. Docker/UDP and host
|
||||
activation before images, collections and draining. Host deletion tests cover
|
||||
ownership guards, interrupted cleanup, uninstall receipts, credential revocation
|
||||
and retention of controller uploads/history. Docker/systemd/UDP and host
|
||||
observations are simulated in these tests.
|
||||
|
||||
A real Debian/Docker/Plutonium acceptance run is still required before production
|
||||
@@ -165,6 +242,10 @@ coverage is not evidence that a real T5/T6 server has been deployed here. Run:
|
||||
agent activation, and during file activation before restart; verify that the
|
||||
agent holds disruption.
|
||||
- Drain occupied lobbies and verify shared libraries survive their retirement.
|
||||
- Delete a drained host; verify its game copies, data directory, agent and cleanup
|
||||
services/configuration/executables are removed, controller uploads/history stay,
|
||||
and unrelated Docker workloads continue. Repeat with connection loss during
|
||||
cleanup and after the final receipt, and a reboot during data removal.
|
||||
- Corrupt/interrupted library downloads, resumed transfer and verified promotion.
|
||||
- Controller restart, agent restart and network loss before execution, during
|
||||
execution and after execution but before acknowledgement. Verify the same
|
||||
|
||||
@@ -10,10 +10,10 @@ export interface HostLibraryCache {
|
||||
}
|
||||
export interface GameLibraryVersion {
|
||||
id: string; libraryId: string; game: GameLibraryGame; version: string;
|
||||
description: string; dlc: string[]; state: GameLibraryState; sizeBytes: number;
|
||||
description: string; state: GameLibraryState; sizeBytes: number;
|
||||
fileCount: number; manifestSha256: string | null; files: GameFileEntry[];
|
||||
createdAt: string; publishedAt: string | null; storageLocation: string;
|
||||
sourceHostId: string; sourceVolume: string; error: string | null; operationId: string | null;
|
||||
sourceHostId: string | null; sourceVolume: string | null; error: string | null; operationId: string | null;
|
||||
caches: HostLibraryCache[];
|
||||
}
|
||||
export interface GameLibrary {
|
||||
|
||||
+24
-1
@@ -1,7 +1,28 @@
|
||||
export type HostMode = 'active' | 'draining' | 'disabled';
|
||||
export type HostReadiness = 'preparing' | 'ready' | 'degraded' | 'draining' | 'disabled';
|
||||
export type HostReadiness = 'preparing' | 'needs-runtime' | 'needs-game-files' | 'ready' | 'degraded' | 'draining' | 'disabled' | 'deleting';
|
||||
export type HostOperationState = 'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown';
|
||||
|
||||
export function hostReadinessLabel(readiness: HostReadiness): string {
|
||||
return { preparing: 'Preparing', 'needs-runtime': 'Needs runtime', 'needs-game-files': 'Needs game files',
|
||||
ready: 'Ready', degraded: 'Degraded', draining: 'Draining', disabled: 'Disabled', deleting: 'Deleting' }[readiness];
|
||||
}
|
||||
|
||||
export interface HostDeletion { allowed: boolean; blockers: string[] }
|
||||
export interface HostDecommission {
|
||||
operationId: string; stage: 'queued' | 'running' | 'cleaned' | 'failed' | 'completed';
|
||||
requestedAt: string; updatedAt: string; error: string | null; retryable: boolean;
|
||||
}
|
||||
export function hostDeletionMessage(host: Pick<Host, 'connected' | 'decommission'>): string {
|
||||
if (!host.decommission) return '';
|
||||
switch (host.decommission.stage) {
|
||||
case 'queued': return host.connected ? 'Host cleanup is queued.' : 'Waiting for the host to reconnect before removing its files and agent.';
|
||||
case 'running': return 'Removing host files, cached game libraries, and the agent installation.';
|
||||
case 'cleaned': return 'Host data is removed. Waiting for confirmation that the agent has uninstalled itself.';
|
||||
case 'failed': return 'Cleanup needs attention. Complete removal has not been confirmed.';
|
||||
case 'completed': return 'Host files and agent removal confirmed.';
|
||||
}
|
||||
}
|
||||
|
||||
export interface HostLibrary { id: string; state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; progress?: number; error?: string }
|
||||
export interface HostInventory {
|
||||
hostname?: string; agentVersion?: string; protocolVersion?: number; os?: string; architecture?: string;
|
||||
@@ -9,6 +30,7 @@ export interface HostInventory {
|
||||
storageTotal?: number; storageAvailable?: number; dockerVersion?: string; dockerRunning?: boolean;
|
||||
directoriesWritable?: boolean; runtimeImages?: string[]; libraries?: HostLibrary[];
|
||||
currentOperation?: string | null; [key: string]: unknown;
|
||||
capabilities?: string[];
|
||||
}
|
||||
export interface Host {
|
||||
id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number;
|
||||
@@ -16,6 +38,7 @@ export interface Host {
|
||||
lastHeartbeat: string | null; connected: boolean; connectionState: 'online' | 'offline';
|
||||
readiness: HostReadiness; readinessReasons: string[]; inventory: HostInventory;
|
||||
enrolled: boolean; credentialActive: boolean;
|
||||
decommission: HostDecommission | null;
|
||||
}
|
||||
export interface HostOperation {
|
||||
id: string; hostId: string; kind: string; instanceId: string | null; generation: number | null;
|
||||
|
||||
@@ -7,7 +7,8 @@ import { assertRegularPath, parseByteRange } from './storage.js';
|
||||
|
||||
export function gameFileError(cause: unknown): Response {
|
||||
const status = typeof cause === 'object' && cause !== null && 'status' in cause && typeof cause.status === 'number' ? cause.status : 400;
|
||||
return json({ error: cause instanceof Error ? cause.message : 'Game file request failed.' }, { status });
|
||||
const receivedBytes = typeof cause === 'object' && cause !== null && 'receivedBytes' in cause && Number.isSafeInteger(cause.receivedBytes) ? cause.receivedBytes : undefined;
|
||||
return json({ error: cause instanceof Error ? cause.message : 'Game file request failed.', ...(receivedBytes === undefined ? {} : { receivedBytes }) }, { status });
|
||||
}
|
||||
export async function readGameJson(request: Request, maxBytes = 64 * 1024): Promise<Record<string, unknown>> {
|
||||
if (!request.body) throw new Error('A JSON request body is required.');
|
||||
|
||||
@@ -6,6 +6,7 @@ import { config } from '../config.js';
|
||||
import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hosts/index.js';
|
||||
import { ensureGameFileSchema } from './schema.js';
|
||||
import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js';
|
||||
import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock } from './uploads.js';
|
||||
import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState } from '../../game-files.js';
|
||||
|
||||
function database() { const handle = db(); ensureGameFileSchema(handle); return handle; }
|
||||
@@ -17,10 +18,10 @@ const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ hostId: row.host_id
|
||||
function versionFromRow(row: VersionRow): GameLibraryVersion {
|
||||
const files = JSON.parse(row.manifest) as GameFileEntry[];
|
||||
return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version,
|
||||
description: row.description, dlc: JSON.parse(row.dlc), state: row.state, sizeBytes: row.size_bytes,
|
||||
files, fileCount: files.length, manifestSha256: row.manifest_sha256, createdAt: row.created_at,
|
||||
publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id,
|
||||
sourceVolume: row.source_volume, error: row.error, operationId: row.operation_id,
|
||||
description: row.description, state: row.state, sizeBytes: row.size_bytes,
|
||||
files, fileCount: files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count, manifestSha256: row.manifest_sha256, createdAt: row.created_at,
|
||||
publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id || null,
|
||||
sourceVolume: row.source_volume || null, error: row.error, operationId: row.operation_id,
|
||||
caches: (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) };
|
||||
}
|
||||
export function getLibraryVersion(id: string): GameLibraryVersion | null {
|
||||
@@ -60,6 +61,112 @@ function textField(input: unknown, label: string, max: number, required = true):
|
||||
if (typeof input !== 'string' || input.length > max || (required && !input.trim())) throw new Error(`Provide a valid ${label}.`);
|
||||
return input.trim();
|
||||
}
|
||||
|
||||
/** Register a folder on the controller; no host or Docker volume is needed. */
|
||||
export function createLibraryUpload(input: Record<string, unknown>): GameLibraryVersion {
|
||||
const game = input.game;
|
||||
if (game !== 't5' && game !== 't6') throw new Error('Choose T5 or T6 game files.');
|
||||
const name = textField(input.name, 'library name', 160);
|
||||
const description = textField(input.description ?? '', 'description', 4000, false);
|
||||
const version = textField(input.version, 'version label', 120);
|
||||
const files = validateUploadPlan(input.files);
|
||||
const handle = database();
|
||||
const id = randomUUID(); const now = new Date().toISOString();
|
||||
handle.transaction(() => {
|
||||
let library = handle.prepare('SELECT id,game FROM game_file_libraries WHERE name=?').get(name) as { id: string; game: string } | undefined;
|
||||
if (library && library.game !== game) throw new Error('This library name is already used for another game.');
|
||||
if (!library) {
|
||||
library = { id: randomUUID(), game };
|
||||
handle.prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)').run(library.id, name, game, description, now);
|
||||
}
|
||||
if (handle.prepare('SELECT 1 FROM game_file_versions WHERE library_id=? AND version=?').get(library.id, version)) throw new Error('This version label already exists. Choose a new version.');
|
||||
// Keep legacy non-null columns for existing databases and published versions.
|
||||
handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,size_bytes,created_at,storage_location,source_host_id,source_volume)
|
||||
VALUES(?,?,?,?,'[]','importing',?,?,?,'','')`).run(id, library.id, version, description, files.reduce((sum, file) => sum + file.size, 0), now, join(gameFilesRoot(), game, id));
|
||||
const insert = handle.prepare('INSERT INTO game_file_upload_entries(version_id,path,size,last_modified) VALUES(?,?,?,?)');
|
||||
for (const file of files) insert.run(id, file.path, file.size, file.lastModified);
|
||||
})();
|
||||
return getLibraryVersion(id)!;
|
||||
}
|
||||
|
||||
type UploadEntry = { path: string; size: number; lastModified: number };
|
||||
function requireBrowserUpload(id: string): GameLibraryVersion {
|
||||
const version = getLibraryVersion(id);
|
||||
if (!version || version.sourceHostId !== null || !database().prepare('SELECT 1 FROM game_file_upload_entries WHERE version_id=? LIMIT 1').get(id)) {
|
||||
throw Object.assign(new Error('Folder upload not found.'), { status: 404 });
|
||||
}
|
||||
return version;
|
||||
}
|
||||
function uploadEntries(id: string): UploadEntry[] {
|
||||
return database().prepare('SELECT path,size,last_modified AS lastModified FROM game_file_upload_entries WHERE version_id=? ORDER BY path').all(id) as UploadEntry[];
|
||||
}
|
||||
async function uploadStorageRoot(version: GameLibraryVersion): Promise<string> {
|
||||
// Recover publication interrupted after the atomic rename but before its DB commit.
|
||||
try { if ((await stat(version.storageLocation)).isDirectory()) return version.storageLocation; }
|
||||
catch (cause) { if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause; }
|
||||
return stagingRoot(version.id);
|
||||
}
|
||||
export async function browserUploadStatus(id: string) {
|
||||
const version = requireBrowserUpload(id);
|
||||
const status = version.state === 'ready' ? 'published' as const : publishing.has(id) ? 'verifying' as const : version.state === 'failed' ? 'failed' as const : 'pending' as const;
|
||||
const root = await uploadStorageRoot(version);
|
||||
const entries = uploadEntries(id);
|
||||
const files = [];
|
||||
// Avoid opening thousands of files concurrently for a large folder.
|
||||
for (const file of entries) files.push({ ...file, ...(status === 'published' ? { receivedBytes: file.size, complete: true } : await getUploadFileProgress(root, file)) });
|
||||
const { files: _manifest, ...summary } = version;
|
||||
return { status, chunkSize: MAX_UPLOAD_CHUNK_BYTES, files, version: summary, error: version.error };
|
||||
}
|
||||
export async function uploadBrowserFile(id: string, path: string, chunk: { offset: number; size: number; sha256: string }, body: ReadableStream<Uint8Array> | null) {
|
||||
validateGamePath(path);
|
||||
const checkWritable = () => {
|
||||
const version = requireBrowserUpload(id);
|
||||
if (version.state === 'ready' || version.manifestSha256 || publishing.has(id)) throw Object.assign(new Error('This upload is no longer writable.'), { status: 409 });
|
||||
};
|
||||
checkWritable();
|
||||
const file = database().prepare('SELECT path,size,last_modified AS lastModified FROM game_file_upload_entries WHERE version_id=? AND path=?').get(id, path) as UploadEntry | undefined;
|
||||
if (!file) throw new Error('File is not listed in this upload.');
|
||||
const progress = await receiveUploadChunk(stagingRoot(id), file, chunk, body, checkWritable);
|
||||
database().prepare("UPDATE game_file_versions SET state='importing',error=NULL WHERE id=? AND state IN ('importing','failed')").run(id);
|
||||
return progress;
|
||||
}
|
||||
export async function publishBrowserUpload(id: string): Promise<GameLibraryVersion> {
|
||||
const version = requireBrowserUpload(id);
|
||||
if (version.state === 'ready') return version;
|
||||
const existing = publishing.get(id); if (existing) return existing;
|
||||
const task = withUploadLock(stagingRoot(id), async () => {
|
||||
const root = await uploadStorageRoot(version);
|
||||
const files = await hashUploadedFiles(root, uploadEntries(id));
|
||||
const checksum = manifestChecksum(files);
|
||||
if (version.manifestSha256 && checksum !== version.manifestSha256) throw new Error('Stored files no longer match the verified manifest.');
|
||||
database().transaction(() => {
|
||||
database().prepare("UPDATE game_file_versions SET state='importing',manifest=?,manifest_sha256=?,error=NULL WHERE id=? AND state IN ('importing','failed')").run(JSON.stringify(files), checksum, id);
|
||||
database().prepare('DELETE FROM game_file_entries WHERE version_id=?').run(id);
|
||||
const insert = database().prepare('INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)');
|
||||
for (const file of files) insert.run(id, file.path, file.size, file.sha256);
|
||||
})();
|
||||
if (root !== version.storageLocation) {
|
||||
await mkdir(dirname(version.storageLocation), { recursive: true });
|
||||
await rename(root, version.storageLocation);
|
||||
}
|
||||
database().prepare("UPDATE game_file_versions SET state='ready',published_at=?,error=NULL WHERE id=? AND state='importing'").run(new Date().toISOString(), id);
|
||||
return getLibraryVersion(id)!;
|
||||
});
|
||||
publishing.set(id, task);
|
||||
try { return await task; } finally { publishing.delete(id); }
|
||||
}
|
||||
export async function requestBrowserPublication(id: string) {
|
||||
const version = requireBrowserUpload(id);
|
||||
if (version.state !== 'ready' && !publishing.has(id)) {
|
||||
database().prepare("UPDATE game_file_versions SET state='importing',error=NULL WHERE id=? AND state IN ('importing','failed')").run(id);
|
||||
void publishBrowserUpload(id).catch(cause => {
|
||||
database().prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'")
|
||||
.run((cause instanceof Error ? cause.message : 'Folder publication failed.').slice(0, 2000), id);
|
||||
});
|
||||
}
|
||||
return browserUploadStatus(id);
|
||||
}
|
||||
|
||||
export function createLibraryImport(input: Record<string, unknown>): GameLibraryVersion {
|
||||
const game = input.game;
|
||||
if (game !== 't5' && game !== 't6') throw new Error('Choose T5 or T6 game files.');
|
||||
@@ -69,9 +176,6 @@ export function createLibraryImport(input: Record<string, unknown>): GameLibrary
|
||||
const name = textField(input.name, 'library name', 160);
|
||||
const description = textField(input.description ?? '', 'description', 4000, false);
|
||||
const version = textField(input.version, 'version label', 120);
|
||||
const dlcInput = input.dlc ?? [];
|
||||
if (!Array.isArray(dlcInput) || dlcInput.length > 64) throw new Error('DLC must be a list of content names.');
|
||||
const dlc = dlcInput.map(value => textField(value, 'DLC name', 120));
|
||||
const sourceVolume = game === 't5' ? 't5_game_files' : 't6_game_files';
|
||||
const handle = database();
|
||||
let library = handle.prepare('SELECT id,game FROM game_file_libraries WHERE name=?').get(name) as { id: string; game: string } | undefined;
|
||||
@@ -84,7 +188,7 @@ export function createLibraryImport(input: Record<string, unknown>): GameLibrary
|
||||
}
|
||||
if (handle.prepare('SELECT 1 FROM game_file_versions WHERE library_id=? AND version=?').get(library.id, version)) throw new Error('This version label already exists. Choose a new version.');
|
||||
handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume)
|
||||
VALUES(?,?,?,?,?,'importing',?,?,?,?)`).run(id, library.id, version, description, JSON.stringify(dlc), now, join(gameFilesRoot(), game, id), hostId, sourceVolume);
|
||||
VALUES(?,?,?,?,?,'importing',?,?,?,?)`).run(id, library.id, version, description, '[]', now, join(gameFilesRoot(), game, id), hostId, sourceVolume);
|
||||
})();
|
||||
try {
|
||||
const operation = queueHostOperation(hostId, 'library.import', {
|
||||
|
||||
@@ -29,6 +29,11 @@ export function ensureGameFileSchema(handle: Database.Database): void {
|
||||
version_id TEXT NOT NULL REFERENCES game_file_versions(id), path TEXT NOT NULL,
|
||||
size INTEGER NOT NULL, sha256 TEXT NOT NULL, PRIMARY KEY(version_id,path)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS game_file_upload_entries (
|
||||
version_id TEXT NOT NULL REFERENCES game_file_versions(id), path TEXT NOT NULL,
|
||||
size INTEGER NOT NULL, last_modified INTEGER NOT NULL,
|
||||
PRIMARY KEY(version_id,path)
|
||||
);
|
||||
INSERT OR IGNORE INTO game_file_entries(version_id,path,size,sha256)
|
||||
SELECT v.id,json_extract(f.value,'$.path'),json_extract(f.value,'$.size'),json_extract(f.value,'$.sha256')
|
||||
FROM game_file_versions v,json_each(v.manifest) f;
|
||||
|
||||
@@ -63,13 +63,18 @@ export async function assertRegularPath(root: string, path: string, allowMissing
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function verifyGameFile(root: string, file: GameFileEntry): Promise<void> {
|
||||
/** Compute a file digest without buffering game archives in memory. */
|
||||
export async function hashGameFile(root: string, file: Pick<GameFileEntry, 'path' | 'size'>): Promise<string> {
|
||||
const path = await assertRegularPath(root, file.path);
|
||||
const info = await stat(path);
|
||||
if (info.size !== file.size) throw new Error(`Size verification failed for ${file.path}.`);
|
||||
const hash = createHash('sha256');
|
||||
for await (const chunk of createReadStream(path)) hash.update(chunk);
|
||||
if (hash.digest('hex') !== file.sha256) throw new Error(`Checksum verification failed for ${file.path}.`);
|
||||
return hash.digest('hex');
|
||||
}
|
||||
|
||||
export async function verifyGameFile(root: string, file: GameFileEntry): Promise<void> {
|
||||
if (await hashGameFile(root, file) !== file.sha256) throw new Error(`Checksum verification failed for ${file.path}.`);
|
||||
}
|
||||
|
||||
/** Stream to a private temporary file; publish only after byte count and SHA256 match. */
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { constants, createReadStream, createWriteStream } from 'node:fs';
|
||||
import { lstat, mkdir, open, rm } from 'node:fs/promises';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { Readable, Transform } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import type { GameFileEntry } from '../../game-files.js';
|
||||
import { assertRegularPath, hashGameFile, validateManifest } from './storage.js';
|
||||
|
||||
export const MAX_UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
export interface UploadFilePlan {
|
||||
path: string;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
}
|
||||
|
||||
export interface UploadFileProgress {
|
||||
receivedBytes: number;
|
||||
complete: boolean;
|
||||
}
|
||||
|
||||
export class UploadOffsetError extends Error {
|
||||
readonly status = 409;
|
||||
readonly receivedBytes: number;
|
||||
constructor(receivedBytes: number) {
|
||||
super(`Upload offset does not match the stored file. Resume at byte ${receivedBytes}.`);
|
||||
this.name = 'UploadOffsetError';
|
||||
this.receivedBytes = receivedBytes;
|
||||
}
|
||||
}
|
||||
|
||||
/** Apply the same path, case, size and file/directory collision rules as published manifests. */
|
||||
export function validateUploadPlan(input: unknown): UploadFilePlan[] {
|
||||
if (!Array.isArray(input) || !input.length || input.length > 200_000) throw new Error('Provide a folder containing 1–200000 files.');
|
||||
const modified = new Map<string, number>();
|
||||
const manifest = validateManifest(input.map(raw => {
|
||||
if (!raw || typeof raw !== 'object') throw new Error('Invalid upload file.');
|
||||
const lastModified = raw.lastModified === undefined ? 0 : raw.lastModified;
|
||||
if (!Number.isSafeInteger(lastModified) || lastModified < 0) throw new Error('Invalid file modification time.');
|
||||
modified.set(raw.path, lastModified);
|
||||
return { path: raw.path, size: raw.size, sha256: '0'.repeat(64) };
|
||||
}));
|
||||
return manifest.map(file => ({ path: file.path, size: file.size, lastModified: modified.get(file.path)! }));
|
||||
}
|
||||
|
||||
const uploadLocks = new Map<string, Promise<void>>();
|
||||
|
||||
/** Serialize transfers and final publication for one import; a rejected operation never poisons the lock. */
|
||||
export async function withUploadLock<T>(root: string, callback: () => Promise<T> | T): Promise<T> {
|
||||
const key = resolve(root);
|
||||
const previous = uploadLocks.get(key) ?? Promise.resolve();
|
||||
let release!: () => void;
|
||||
const current = new Promise<void>(done => { release = done; });
|
||||
uploadLocks.set(key, current);
|
||||
await previous;
|
||||
try { return await callback(); }
|
||||
finally {
|
||||
release();
|
||||
if (uploadLocks.get(key) === current) uploadLocks.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
/** File size is the resume checkpoint, including after a controller restart during an append. */
|
||||
export async function getUploadFileProgress(root: string, file: Pick<UploadFilePlan, 'path' | 'size'>): Promise<UploadFileProgress> {
|
||||
const path = await assertRegularPath(root, file.path, true);
|
||||
try {
|
||||
const info = await lstat(path);
|
||||
if (!info.isFile() || info.isSymbolicLink()) throw new Error('Game storage contains a non-regular path.');
|
||||
if (info.size > file.size) throw new Error(`Stored upload exceeds the declared size of ${file.path}.`);
|
||||
return { receivedBytes: info.size, complete: info.size === file.size };
|
||||
} catch (cause) {
|
||||
if ((cause as NodeJS.ErrnoException).code === 'ENOENT') return { receivedBytes: 0, complete: false };
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify one bounded request before touching the destination. Only verified bytes may be
|
||||
* appended, so a process interruption during the append can resume at the actual file size.
|
||||
* beforeWrite runs under the lock: callers must recheck that the import is still writable.
|
||||
*/
|
||||
export async function receiveUploadChunk(
|
||||
root: string,
|
||||
file: Pick<UploadFilePlan, 'path' | 'size'>,
|
||||
chunk: { offset: number; size: number; sha256: string },
|
||||
body: ReadableStream<Uint8Array> | null,
|
||||
beforeWrite?: () => Promise<void> | void
|
||||
): Promise<UploadFileProgress> {
|
||||
if (!Number.isSafeInteger(chunk.offset) || chunk.offset < 0 || !Number.isSafeInteger(chunk.size)
|
||||
|| chunk.size < 0 || chunk.size > MAX_UPLOAD_CHUNK_BYTES || chunk.offset + chunk.size > file.size
|
||||
|| !Number.isSafeInteger(chunk.offset + chunk.size) || !/^[a-f0-9]{64}$/.test(chunk.sha256)
|
||||
|| (chunk.size === 0 && file.size !== 0)) {
|
||||
throw new Error('Invalid upload chunk size, offset or SHA256.');
|
||||
}
|
||||
if (!body && chunk.size !== 0) throw new Error('A file chunk body is required.');
|
||||
|
||||
return withUploadLock(root, async () => {
|
||||
await beforeWrite?.();
|
||||
const progress = await getUploadFileProgress(root, file);
|
||||
if (chunk.offset !== progress.receivedBytes) throw new UploadOffsetError(progress.receivedBytes);
|
||||
const destination = await assertRegularPath(root, file.path, true);
|
||||
// Keep transient chunks outside the tree that will become the immutable library.
|
||||
const chunkRoot = `${resolve(root)}.chunks`;
|
||||
const temporary = await assertRegularPath(chunkRoot, randomUUID(), true);
|
||||
await mkdir(dirname(temporary), { recursive: true });
|
||||
let received = 0;
|
||||
const hash = createHash('sha256');
|
||||
const verifier = new Transform({ transform(data, _encoding, done) {
|
||||
received += data.length;
|
||||
if (received > chunk.size) return done(new Error('Upload exceeds the declared chunk size.'));
|
||||
hash.update(data); done(null, data);
|
||||
} });
|
||||
try {
|
||||
await pipeline(body ? Readable.fromWeb(body as never) : Readable.from([]), verifier,
|
||||
createWriteStream(temporary, { flags: 'wx', mode: 0o600, flush: true }));
|
||||
if (received !== chunk.size || hash.digest('hex') !== chunk.sha256) throw new Error('Upload chunk checksum or size mismatch.');
|
||||
|
||||
await mkdir(dirname(destination), { recursive: true });
|
||||
await assertRegularPath(root, file.path, true);
|
||||
const target = await open(destination, constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW, 0o600);
|
||||
try {
|
||||
const info = await target.stat();
|
||||
if (!info.isFile()) throw new Error('Game storage contains a non-regular path.');
|
||||
if (info.size !== chunk.offset) throw new UploadOffsetError(info.size);
|
||||
let position = chunk.offset;
|
||||
try {
|
||||
for await (const data of createReadStream(temporary)) {
|
||||
const bytes = data as Buffer;
|
||||
let written = 0;
|
||||
while (written < bytes.length) {
|
||||
const result = await target.write(bytes, written, bytes.length - written, position);
|
||||
if (result.bytesWritten === 0) throw new Error('Unable to append the upload chunk.');
|
||||
written += result.bytesWritten;
|
||||
position += result.bytesWritten;
|
||||
}
|
||||
}
|
||||
await target.sync();
|
||||
} catch (cause) {
|
||||
await target.truncate(chunk.offset);
|
||||
await target.sync();
|
||||
throw cause;
|
||||
}
|
||||
} finally { await target.close(); }
|
||||
return { receivedBytes: chunk.offset + chunk.size, complete: chunk.offset + chunk.size === file.size };
|
||||
} finally { await rm(temporary, { force: true }); }
|
||||
});
|
||||
}
|
||||
|
||||
/** Caller holds withUploadLock through hashing and publication to prevent concurrent writes. */
|
||||
export async function hashUploadedFiles(root: string, files: ReadonlyArray<Pick<UploadFilePlan, 'path' | 'size'>>): Promise<GameFileEntry[]> {
|
||||
const manifest: GameFileEntry[] = [];
|
||||
for (const file of files) manifest.push({ path: file.path, size: file.size, sha256: await hashGameFile(root, file) });
|
||||
return validateManifest(manifest);
|
||||
}
|
||||
+133
-10
@@ -1,8 +1,8 @@
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
|
||||
import { isIP } from 'node:net';
|
||||
import { db } from '../db/index.js';
|
||||
import { ensureHostSchema } from './schema.js';
|
||||
import type { Host, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
|
||||
import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
|
||||
|
||||
export const AGENT_PROTOCOL_VERSION = 1;
|
||||
export const HEARTBEAT_MAX_AGE_MS = 90_000;
|
||||
@@ -11,6 +11,14 @@ const digest = (token: string) => createHash('sha256').update(token).digest('hex
|
||||
function database() { const handle = db(); ensureHostSchema(handle); return handle; }
|
||||
type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number };
|
||||
const hostSql = `SELECT h.*, EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id = h.id AND c.revoked_at IS NULL) AS credential_active FROM ec_hosts h`;
|
||||
type DecommissionRow = { host_id: string; operation_id: string; token_hash: string; stage: HostDecommission['stage']; requested_at: string; updated_at: string; cleaned_at: string | null; error: string | null };
|
||||
function decommissionRow(hostId: string): DecommissionRow | undefined {
|
||||
return database().prepare('SELECT * FROM ec_host_decommissions WHERE host_id=?').get(hostId) as DecommissionRow | undefined;
|
||||
}
|
||||
function presentDecommission(row: DecommissionRow | undefined): HostDecommission | null {
|
||||
return row ? { operationId: row.operation_id, stage: row.stage, requestedAt: row.requested_at, updatedAt: row.updated_at,
|
||||
error: row.error, retryable: row.stage === 'failed' && getHostOperation(row.operation_id)?.state === 'failed' } : null;
|
||||
}
|
||||
function hostFromRow(row: HostRow): Host {
|
||||
const inventory: HostInventory = JSON.parse(row.inventory);
|
||||
const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
|
||||
@@ -23,22 +31,123 @@ function hostFromRow(row: HostRow): Host {
|
||||
if (!inventory.directoriesWritable) reasons.push('Managed directories are not writable.');
|
||||
if (typeof inventory.storageAvailable !== 'number' || inventory.storageAvailable < row.minimum_free_bytes) reasons.push('Insufficient available game storage.');
|
||||
const infrastructureReady = reasons.length === 0;
|
||||
if (!inventory.runtimeImages?.length) reasons.push('Prepare a Plutainer runtime image.');
|
||||
if (!inventory.libraries?.some(library => library.state === 'ready')) reasons.push('Download and verify a game-file library.');
|
||||
const runtimeReady = !!inventory.runtimeImages?.length;
|
||||
const libraryReady = !!inventory.libraries?.some(library => library.state === 'ready');
|
||||
if (!runtimeReady) reasons.push('Prepare a Plutainer runtime image.');
|
||||
if (!libraryReady) reasons.push('Download and verify a game-file library.');
|
||||
if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
|
||||
if (row.mode === 'disabled') reasons.push('Host is disabled.');
|
||||
const decommission = presentDecommission(decommissionRow(row.id));
|
||||
if (decommission) reasons.push('Host deletion is in progress; waiting for confirmed cleanup and agent removal.');
|
||||
return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max,
|
||||
bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes,
|
||||
createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline',
|
||||
readiness: row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at || (infrastructureReady && reasons.length) ? 'preparing' : reasons.length ? 'degraded' : 'ready',
|
||||
readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active };
|
||||
readiness: decommission ? 'deleting' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
|
||||
readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active, decommission };
|
||||
}
|
||||
export function listHosts(): Host[] { return (database().prepare(hostSql + ' ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
|
||||
export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
|
||||
export function getHost(id: string): Host | null {
|
||||
const row = database().prepare(hostSql + ' WHERE h.id = ?').get(id) as HostRow | undefined;
|
||||
const row = database().prepare(hostSql + ' WHERE h.id = ? AND h.deleted_at IS NULL').get(id) as HostRow | undefined;
|
||||
return row ? hostFromRow(row) : null;
|
||||
}
|
||||
function requireHost(id: string): Host { const host = getHost(id); if (!host) throw new Error('Host was not found.'); return host; }
|
||||
|
||||
export class HostDeletionError extends Error {
|
||||
constructor(message: string, readonly status: 400 | 401 | 404 | 409, readonly blockers: string[] = []) { super(message); }
|
||||
}
|
||||
|
||||
/** Controller records can be removed only after all remote instance ownership is resolved. */
|
||||
export function hostDeletionStatus(id: string): HostDeletion {
|
||||
const host = getHost(id);
|
||||
if (!host) throw new HostDeletionError('Host not found.', 404);
|
||||
if (host.decommission && !host.decommission.retryable) return { allowed: true, blockers: [] };
|
||||
const handle = database();
|
||||
const blockers: string[] = [];
|
||||
if (host.enrolled && !host.inventory.capabilities?.includes('host.decommission')) blockers.push('Update this host agent before deleting it: generate an installer from Agent access, rerun it on this host, and wait for a fresh heartbeat.');
|
||||
if (host.enrolled && !host.credentialActive) blockers.push('Re-enroll this host so its agent can receive and complete the cleanup request.');
|
||||
const instances = handle.prepare('SELECT server FROM native_instances WHERE host_id=? AND retired_at IS NULL ORDER BY server').all(id) as { server: string }[];
|
||||
if (instances.length) {
|
||||
const names = instances.slice(0, 5).map(instance => instance.server).join(', ') + (instances.length > 5 ? ', …' : '');
|
||||
blockers.push(`Remove the ${instances.length} server instance${instances.length === 1 ? '' : 's'} assigned to this host first (${names}). Queued and failed deployments also require removal.`);
|
||||
}
|
||||
const ports = handle.prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE host_id=?').get(id) as { count: number };
|
||||
if (ports.count && !instances.length) blockers.push(`This host still has ${ports.count} reserved game port${ports.count === 1 ? '' : 's'}. Finish or reconcile the corresponding server removals first.`);
|
||||
// A pending command may have reached the agent before its acknowledgement.
|
||||
// Only commands that have never been delivered are safe to cancel locally.
|
||||
const operations = handle.prepare(`SELECT kind,state FROM ec_host_operations WHERE host_id=?
|
||||
AND (state IN ('accepted','running','unknown') OR (state='pending' AND last_delivered_at IS NOT NULL))
|
||||
ORDER BY created_at`).all(id) as { kind: string; state: string }[];
|
||||
if (operations.length) {
|
||||
const work = operations.slice(0, 5).map(operation => `${operation.kind}: ${operation.state}`).join(', ') + (operations.length > 5 ? ', …' : '');
|
||||
blockers.push(`${operations.length} delivered operation${operations.length === 1 ? ' is' : 's are'} unresolved (${work}). Let the agent reconnect and finish or reconcile this work before deleting the host.`);
|
||||
}
|
||||
return { allowed: blockers.length === 0, blockers };
|
||||
}
|
||||
|
||||
/** Keep the host visible and authenticated until its independent cleanup worker proves removal. */
|
||||
export function deleteHost(id: string): { deleted: boolean; operationId?: string } {
|
||||
const handle = database();
|
||||
return handle.transaction(() => {
|
||||
const host = getHost(id);
|
||||
if (!host) throw new HostDeletionError('Host not found.', 404);
|
||||
if (host.decommission && !host.decommission.retryable) return { deleted: false, operationId: host.decommission.operationId };
|
||||
const status = hostDeletionStatus(id);
|
||||
if (!status.allowed) throw new HostDeletionError(status.blockers.join(' '), 409, status.blockers);
|
||||
const timestamp = now();
|
||||
const reason = 'Host deletion requested before this operation was delivered to the agent.';
|
||||
handle.prepare(`INSERT INTO ec_host_operation_events(operation_id,state,created_at,error)
|
||||
SELECT id,'failed',?,? FROM ec_host_operations WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL`).run(timestamp, reason, id);
|
||||
handle.prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL").run(reason, timestamp, id);
|
||||
handle.prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, id);
|
||||
if (!host.enrolled) {
|
||||
handle.prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL").run(timestamp, id);
|
||||
return { deleted: true };
|
||||
}
|
||||
const operationId = randomUUID();
|
||||
const token = 'ec_cleanup_' + randomBytes(32).toString('base64url');
|
||||
const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` };
|
||||
handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at) VALUES(?,?,'host.decommission',?,'pending',?,?)").run(operationId, id, JSON.stringify(payload), timestamp, timestamp);
|
||||
handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp);
|
||||
handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?)
|
||||
ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
|
||||
.run(id, operationId, digest(token), timestamp, timestamp);
|
||||
handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id);
|
||||
return { deleted: false, operationId };
|
||||
}).immediate();
|
||||
}
|
||||
|
||||
export function receiveDecommissionReceipt(operationId: string, token: string, input: Record<string, unknown>): { acknowledged: true } {
|
||||
if (!token || token.length > 200) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
|
||||
return database().transaction(() => {
|
||||
const row = database().prepare('SELECT * FROM ec_host_decommissions WHERE operation_id=?').get(operationId) as DecommissionRow | undefined;
|
||||
if (!row || !timingSafeEqual(Buffer.from(row.token_hash, 'hex'), Buffer.from(digest(token), 'hex'))) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
|
||||
if (!['cleaned', 'completed', 'failed'].includes(input.stage as string)) throw new HostDeletionError('Invalid cleanup receipt stage.', 400);
|
||||
if (row.stage === 'completed') return { acknowledged: true as const };
|
||||
const operation = getHostOperation(operationId)!;
|
||||
if (!['accepted','running','unknown'].includes(operation.state)) throw new HostDeletionError('Cleanup has not been accepted by the host.', 409);
|
||||
const timestamp = now();
|
||||
if (input.stage === 'failed') {
|
||||
database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE host_id=?")
|
||||
.run(typeof input.error === 'string' ? input.error.slice(0, 2000) : 'Host cleanup needs attention. The cleanup worker will retry.', timestamp, row.host_id);
|
||||
return { acknowledged: true as const };
|
||||
}
|
||||
if (input.stage === 'cleaned') {
|
||||
database().prepare("UPDATE ec_host_decommissions SET stage='cleaned',cleaned_at=COALESCE(cleaned_at,?),error=NULL,updated_at=? WHERE host_id=?").run(timestamp, timestamp, row.host_id);
|
||||
return { acknowledged: true as const };
|
||||
}
|
||||
if (!row.cleaned_at) throw new HostDeletionError('The host must confirm file cleanup before confirming agent removal.', 409);
|
||||
database().prepare("UPDATE ec_host_decommissions SET stage='completed',error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id);
|
||||
database().prepare("UPDATE ec_host_operations SET state='completed',result=?,error=NULL,updated_at=? WHERE id=?").run(JSON.stringify({ filesRemoved: true, agentRemoved: true }), timestamp, operationId);
|
||||
database().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'completed',?)").run(operationId, timestamp);
|
||||
database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(timestamp, row.host_id);
|
||||
database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, row.host_id);
|
||||
if (database().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='host_library_caches'").get()) {
|
||||
database().prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id);
|
||||
}
|
||||
database().prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=?").run(timestamp, row.host_id);
|
||||
return { acknowledged: true as const };
|
||||
}).immediate();
|
||||
}
|
||||
function textField(value: unknown, label: string, max = 200): string {
|
||||
if (typeof value !== 'string' || !value.trim() || value.length > max || /[\x00-\x1f]/.test(value)) throw new Error(`${label} must be a nonempty string up to ${max} characters.`);
|
||||
return value.trim();
|
||||
@@ -55,6 +164,7 @@ export function createHost(input: Record<string, unknown>): Host {
|
||||
}
|
||||
export function updateHost(id: string, input: Record<string, unknown>): Host {
|
||||
const host = requireHost(id);
|
||||
if (host.decommission) throw new Error('Host cleanup is in progress; its settings cannot change.');
|
||||
const name = input.name === undefined ? host.name : textField(input.name, 'Host name');
|
||||
const mode = input.mode ?? host.mode;
|
||||
if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.');
|
||||
@@ -70,7 +180,8 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
|
||||
}
|
||||
|
||||
export function createEnrollment(hostId: string, ttlMs = 15 * 60_000): { id: string; hostId: string; token: string; expiresAt: string } {
|
||||
requireHost(hostId);
|
||||
const host = requireHost(hostId);
|
||||
if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup is in progress; enrollment is unavailable.');
|
||||
if (!Number.isInteger(ttlMs) || ttlMs <= 0 || ttlMs > 60 * 60_000) throw new Error('Enrollment lifetime must be at most one hour.');
|
||||
const token = 'ec_enroll_' + randomBytes(32).toString('base64url'); const id = randomUUID();
|
||||
const expiresAt = new Date(Date.now() + ttlMs).toISOString();
|
||||
@@ -96,6 +207,7 @@ export function validateInventory(value: unknown): HostInventory {
|
||||
}
|
||||
for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
|
||||
if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
|
||||
if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
|
||||
if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library =>
|
||||
!library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.');
|
||||
return inventory;
|
||||
@@ -122,7 +234,8 @@ export function authenticateCredential(token: string): Host | null {
|
||||
return getHost(row.host_id);
|
||||
}
|
||||
export function revokeCredentials(hostId: string): void {
|
||||
requireHost(hostId);
|
||||
const host = requireHost(hostId);
|
||||
if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup needs its credentials until removal is confirmed.');
|
||||
database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(now(), hostId);
|
||||
}
|
||||
export function rotateCredential(hostId: string, nextCredential?: string): string {
|
||||
@@ -160,6 +273,8 @@ export function getHostOperation(id: string): HostOperation | null { const row =
|
||||
export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
|
||||
export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation {
|
||||
const host = requireHost(hostId);
|
||||
if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.');
|
||||
if (kind === 'host.decommission') throw new Error('Use the host deletion workflow to request cleanup.');
|
||||
if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
|
||||
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
|
||||
if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.');
|
||||
@@ -195,8 +310,10 @@ export function pendingHostOperations(hostId: string): HostOperation[] {
|
||||
}
|
||||
export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation {
|
||||
return database().transaction(() => {
|
||||
requireHost(hostId);
|
||||
const operation = getHostOperation(id);
|
||||
if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.');
|
||||
if (operation.kind === 'host.decommission' && input.state === 'completed') throw new Error('Host removal requires the cleanup worker completion receipt.');
|
||||
if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.');
|
||||
if (['completed','failed'].includes(operation.state)) return operation;
|
||||
const state = input.state as HostOperationState;
|
||||
@@ -205,6 +322,10 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
|
||||
const result = input.result === undefined ? null : JSON.stringify(input.result);
|
||||
if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.');
|
||||
database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=? WHERE id=?').run(state, result, error, now(), id);
|
||||
if (operation.kind === 'host.decommission') {
|
||||
if (state === 'failed') database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?").run(error ?? 'The agent could not start cleanup. Repair the installation, then retry deletion.', now(), id);
|
||||
else database().prepare("UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')").run(now(), id);
|
||||
}
|
||||
if (operation.state !== state) database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)').run(id, state, now(), error);
|
||||
return getHostOperation(id)!;
|
||||
})();
|
||||
@@ -222,6 +343,7 @@ export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise
|
||||
|
||||
export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean {
|
||||
const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress;
|
||||
if (host.decommission) return false;
|
||||
if (candidate < host.portMin || candidate > host.portMax) return false;
|
||||
const rows = database().prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND protocol=? AND port=?').all(hostId, protocol, candidate) as { instance_id: string; bind_address: string }[];
|
||||
return !rows.some(row => row.instance_id !== instanceId && (row.bind_address === bind || ['0.0.0.0','::'].includes(bind) || ['0.0.0.0','::'].includes(row.bind_address)));
|
||||
@@ -233,6 +355,7 @@ export function nextHostPort(hostId: string, additionalPorts: number[] = [], bin
|
||||
}
|
||||
export function reserveHostPort(hostId: string, instanceId: string, options: { port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): number {
|
||||
const host = requireHost(hostId); const bind = options.bindAddress ?? host.bindAddress; const protocol = options.protocol ?? 'udp';
|
||||
if (host.decommission) throw new Error('Host cleanup is in progress; new port reservations are blocked.');
|
||||
if (!isIP(bind)) throw new Error('Bind address must be an IP address.');
|
||||
if (!['udp','tcp'].includes(protocol)) throw new Error('Port protocol must be UDP or TCP.');
|
||||
if (!instanceId) throw new Error('An immutable instance ID is required.');
|
||||
|
||||
@@ -10,7 +10,7 @@ export function ensureHostSchema(handle: Database.Database): void {
|
||||
port_min INTEGER NOT NULL DEFAULT 28960, port_max INTEGER NOT NULL DEFAULT 28980,
|
||||
bind_address TEXT NOT NULL DEFAULT '0.0.0.0', public_address TEXT NOT NULL DEFAULT '',
|
||||
minimum_free_bytes INTEGER NOT NULL DEFAULT 1073741824,
|
||||
created_at TEXT NOT NULL, enrolled_at TEXT, last_heartbeat TEXT, inventory TEXT NOT NULL DEFAULT '{}'
|
||||
created_at TEXT NOT NULL, enrolled_at TEXT, last_heartbeat TEXT, inventory TEXT NOT NULL DEFAULT '{}', deleted_at TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ec_host_enrollments (
|
||||
id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), token_hash TEXT NOT NULL UNIQUE,
|
||||
@@ -40,8 +40,17 @@ export function ensureHostSchema(handle: Database.Database): void {
|
||||
PRIMARY KEY(host_id,bind_address,protocol,port)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id);
|
||||
CREATE TABLE IF NOT EXISTS ec_host_decommissions (
|
||||
host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id),
|
||||
operation_id TEXT NOT NULL UNIQUE REFERENCES ec_host_operations(id),
|
||||
token_hash TEXT NOT NULL, stage TEXT NOT NULL DEFAULT 'queued'
|
||||
CHECK(stage IN ('queued','running','cleaned','failed','completed')),
|
||||
requested_at TEXT NOT NULL, updated_at TEXT NOT NULL, cleaned_at TEXT, error TEXT
|
||||
);
|
||||
`);
|
||||
const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[];
|
||||
if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');
|
||||
const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[];
|
||||
if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
|
||||
initialized.add(handle);
|
||||
}
|
||||
|
||||
@@ -204,7 +204,7 @@ export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blue
|
||||
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All 20 lobbies for this map are occupied or reserved.' },
|
||||
{ label: 'Game port', state: port === null ? 'blocked' : 'pass', detail: port === null ? 'No free game port. Free capacity before launching.' : `${port} / UDP and TCP. Availability is checked again at launch.` },
|
||||
{ label: 'Promoted configuration', state: 'pass', detail: recipe.configSource },
|
||||
{ label: 'Game files and host readiness', state: 'info', detail: 'The host must have the selected game and DLC files. This preview does not probe volumes, images, credentials, or network access.' }
|
||||
{ label: 'Game files and host readiness', state: 'info', detail: 'The host must have the selected game files. This preview does not probe volumes, images, credentials, or network access.' }
|
||||
];
|
||||
const id = randomUUID();
|
||||
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
<script lang="ts">
|
||||
import type { Host, HostDeletion } from '$lib/hosts.js';
|
||||
import Button from './primitives/Button.svelte';
|
||||
import Dialog from './primitives/Dialog.svelte';
|
||||
|
||||
let { open = $bindable(false), host, ondeleted }:
|
||||
{ open?: boolean; host: Pick<Host, 'id' | 'name'> | null; ondeleted: (result: { deleted: boolean; operationId?: string }) => void | Promise<void> } = $props();
|
||||
let deletion = $state<HostDeletion | null>(null);
|
||||
let checking = $state(false); let deleting = $state(false); let error = $state('');
|
||||
|
||||
async function inspect(id: string, signal?: AbortSignal) {
|
||||
checking = true; deletion = null; error = '';
|
||||
try {
|
||||
const response = await fetch(`/api/hosts/${encodeURIComponent(id)}`, { signal });
|
||||
const body = await response.json();
|
||||
if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not check this host.');
|
||||
if (!signal?.aborted) deletion = body.deletion;
|
||||
} catch (cause) { if (!signal?.aborted) error = (cause as Error).message; }
|
||||
finally { if (!signal?.aborted) checking = false; }
|
||||
}
|
||||
$effect(() => {
|
||||
if (!open || !host) return;
|
||||
const controller = new AbortController();
|
||||
void inspect(host.id, controller.signal);
|
||||
return () => controller.abort();
|
||||
});
|
||||
async function remove() {
|
||||
if (!host || deleting || checking || !deletion?.allowed) return;
|
||||
const id = host.id;
|
||||
deleting = true; error = '';
|
||||
try {
|
||||
const response = await fetch(`/api/hosts/${encodeURIComponent(id)}`, { method: 'DELETE' });
|
||||
const body = await response.json();
|
||||
if (!response.ok) {
|
||||
await inspect(id);
|
||||
throw new Error(body.error ?? body.message ?? 'Could not delete this host.');
|
||||
}
|
||||
open = false;
|
||||
await ondeleted(body);
|
||||
} catch (cause) { error = (cause as Error).message; }
|
||||
finally { deleting = false; }
|
||||
}
|
||||
</script>
|
||||
|
||||
<Dialog bind:open title={`Delete ${host?.name ?? 'host'}?`} busy={deleting}
|
||||
description="Permanently remove Electric Cherry's files from this host, including its downloaded game files, then uninstall the agent.">
|
||||
<div class="content">
|
||||
<p>This deletes the host's game-library copies, server files, agent data, credentials, and service installation. Electric Cherry's uploaded libraries on the controller and its operation history are kept.</p>
|
||||
<p>The host remains visible until cleanup is confirmed. Offline hosts wait for reconnection. Unsent commands will be cancelled.</p>
|
||||
{#if checking}<p role="status">Checking servers and unfinished operations…</p>
|
||||
{:else if deletion && !deletion.allowed}<div role="status" class="blocked"><strong>This host still has work to finish:</strong><ul>{#each deletion.blockers as blocker}<li>{blocker}</li>{/each}</ul><p>Open <a href={`/hosts/${host?.id}`} onclick={() => open = false}>host details</a> to review its operations or drain its servers.</p></div>
|
||||
{:else if deletion?.allowed}<p>Cleanup cannot be undone once it starts. Adding this machine again requires installing and enrolling the agent again.</p>{/if}
|
||||
{#if error}<p role="alert" class="ui-error">{error}</p>{/if}
|
||||
</div>
|
||||
{#snippet actions()}
|
||||
<Button variant="ghost" disabled={deleting} onclick={() => open = false}>Cancel</Button>
|
||||
{#if !deletion?.allowed}<Button disabled={checking || deleting || !host} onclick={() => { if (host) void inspect(host.id); }}>Check again</Button>{/if}
|
||||
<Button variant="danger" loading={deleting} disabled={checking || !deletion?.allowed} onclick={remove}>Delete host</Button>
|
||||
{/snippet}
|
||||
</Dialog>
|
||||
|
||||
<style>
|
||||
.content { display:grid; gap:16px; } p,li { margin:0; color:var(--text-secondary); font-size:var(--text-sm); line-height:1.65; }
|
||||
.blocked { display:grid; gap:12px; } strong { font-size:var(--text-sm); } ul { margin:0; padding-left:20px; }
|
||||
a { color:var(--accent-primary-hover); } .ui-error { color:var(--state-danger); }
|
||||
</style>
|
||||
@@ -1,6 +1,7 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import type { Host, NativeDeploymentSelection } from '$lib/hosts.js';
|
||||
import { hostReadinessLabel } from '$lib/hosts.js';
|
||||
import type { GameLibrary } from '$lib/game-files.js';
|
||||
import type { ZombieGame } from '$lib/map-aliases.js';
|
||||
let { games, selection = $bindable(), ready = $bindable(false), disabled = false, network = false, onchange }:
|
||||
@@ -26,11 +27,11 @@
|
||||
<div class="title"><h3>Host and game files</h3><button type="button" disabled={disabled || loading} onclick={() => { onchange?.(); void load(); }}>Refresh hosts</button></div>
|
||||
{#if error}<p role="alert" class="error">{error}</p>{/if}
|
||||
{#if loading}<p role="status">Loading host readiness and libraries…</p>{/if}
|
||||
<label>Game host<select bind:value={selection.hostId} disabled={disabled || loading} onchange={() => onchange?.()}><option value="">Choose a host</option>{#each hosts as candidate}<option value={candidate.id} disabled={candidate.readiness !== 'ready'}>{candidate.name} · {candidate.connectionState} · {candidate.readiness}</option>{/each}</select></label>
|
||||
<label>Game host<select bind:value={selection.hostId} disabled={disabled || loading} onchange={() => onchange?.()}><option value="">Choose a host</option>{#each hosts as candidate}<option value={candidate.id} disabled={candidate.readiness !== 'ready'}>{candidate.name} · {candidate.connectionState} · {hostReadinessLabel(candidate.readiness)}</option>{/each}</select></label>
|
||||
{#if !loading && !hosts.some(candidate => candidate.readiness === 'ready')}<p>No deployment-ready hosts. <a href="/hosts" target="_blank" rel="noopener">Set up a host</a>, then refresh.</p>{/if}
|
||||
{#if host}<p>{host.hostname} · ports {host.portMin}–{host.portMax} · {host.publicAddress || host.bindAddress}</p>{/if}
|
||||
{#each games as game}<label>{game === 't5' ? 'Black Ops I' : 'Black Ops II'} game-file version<select bind:value={selection.libraryIds[game]} disabled={disabled || loading || !host} onchange={() => onchange?.()}><option value="">Choose an immutable library version</option>{#each versions.filter(version => version.game === game) as version}<option value={version.id} disabled={!cacheReady(version.id)}>{version.name} · {version.version} · {cacheReady(version.id) ? 'Ready on host' : 'Not cached on host'}</option>{/each}</select></label>{/each}
|
||||
<p><a href="/game-files" target="_blank" rel="noopener">Import or download game files to a host</a>, then refresh. Each launch stays pinned to its selected library version.</p>
|
||||
<p><a href="/game-files" target="_blank" rel="noopener">Upload a game folder or download a version to a host</a>, then refresh. Each launch stays pinned to its selected library version.</p>
|
||||
<label>Plutainer runtime image<input required bind:value={selection.image} disabled={disabled} onchange={() => onchange?.()} /></label>
|
||||
{#if host && !imageReady}<p class="error">Prepare this runtime image from <a href={`/hosts/${host.id}`} target="_blank" rel="noopener">host setup</a> before launch.</p>{/if}
|
||||
{#if network}<details><summary>Network overrides</summary><label>Local bind address<input bind:value={selection.bindAddress} placeholder={host?.bindAddress ?? 'Host default'} disabled={disabled} onchange={() => onchange?.()} /></label><label>Public / NAT address<input bind:value={selection.publicAddress} placeholder={host?.publicAddress || 'Host default'} disabled={disabled} onchange={() => onchange?.()} /></label><label>Public / NAT port<input type="number" min="1" max="65535" bind:value={selection.publicPort} placeholder="Same as local port" disabled={disabled} onchange={() => onchange?.()} /></label><p>Public mapping does not change the local Docker port reservation.</p></details>{/if}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
export interface SelectedGameFile { path: string; file: File }
|
||||
export interface SelectedGameFolder { name: string; files: SelectedGameFile[]; size: number }
|
||||
|
||||
/** Directory inputs include the selected folder in every relative path. Remove
|
||||
* that one component and retain the complete game directory below it. */
|
||||
export function selectGameFolder(selection: Iterable<File>): SelectedGameFolder {
|
||||
const files: SelectedGameFile[] = [];
|
||||
const paths = new Set<string>();
|
||||
let name = '';
|
||||
let size = 0;
|
||||
for (const file of selection) {
|
||||
const [root, ...parts] = file.webkitRelativePath.split('/');
|
||||
if (!root || root === '.' || root === '..' || !parts.length || name && root !== name) throw new Error('Choose one game folder with the folder picker.');
|
||||
name = root;
|
||||
const path = parts.join('/');
|
||||
if (path.length > 1024 || /[\\\x00-\x1f\x7f:]/.test(path) || parts.some(part => !part || part === '.' || part === '..' || /[. ]$/.test(part))) {
|
||||
throw new Error(`This file has an unsupported game path: ${file.webkitRelativePath}`);
|
||||
}
|
||||
const key = path.toLowerCase();
|
||||
if (paths.has(key)) throw new Error(`This folder has duplicate game paths: ${path}`);
|
||||
paths.add(key);
|
||||
size += file.size;
|
||||
if (!Number.isSafeInteger(size)) throw new Error('This folder exceeds the supported upload size.');
|
||||
files.push({ path, file });
|
||||
if (files.length > 200_000) throw new Error('Choose a folder with no more than 200,000 files.');
|
||||
}
|
||||
if (!files.length) throw new Error('The selected folder contains no files.');
|
||||
for (const { path } of files) {
|
||||
const parents = path.toLowerCase().split('/');
|
||||
parents.pop();
|
||||
while (parents.length) {
|
||||
if (paths.has(parents.join('/'))) throw new Error(`This folder uses the same game path for a file and a directory: ${path}`);
|
||||
parents.pop();
|
||||
}
|
||||
}
|
||||
files.sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
|
||||
return { name, files, size };
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { HostDeletionError, receiveDecommissionReceipt } from '$lib/server/hosts/index.js';
|
||||
import { objectBody, requireAgentTransport } from '$lib/server/hosts/http.js';
|
||||
|
||||
/** The uninstall worker retains only this narrowly scoped credential after removing the agent. */
|
||||
export async function POST({ request, params }: { request: Request; params: { id: string } }) {
|
||||
requireAgentTransport(request);
|
||||
const token = request.headers.get('authorization')?.match(/^Bearer ([A-Za-z0-9_-]+)$/i)?.[1] ?? '';
|
||||
try { return json(receiveDecommissionReceipt(params.id, token, await objectBody(request, 8192)), { headers: { 'Cache-Control': 'no-store' } }); }
|
||||
catch (cause) {
|
||||
if (cause instanceof HostDeletionError) return json({ error: cause.message }, { status: cause.status });
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,10 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { createLibraryImport, libraryOverview } from '$lib/server/game-files/index.js';
|
||||
import { createLibraryUpload, libraryOverview } from '$lib/server/game-files/index.js';
|
||||
import { gameFileError, readGameJson } from '$lib/server/game-files/http.js';
|
||||
import { requireManagementOrigin } from '$lib/server/hosts/http.js';
|
||||
|
||||
export function GET() { return json(libraryOverview()); }
|
||||
export async function POST({ request }: { request: Request }) {
|
||||
try { requireManagementOrigin(request); return json({ version: createLibraryImport(await readGameJson(request)) }, { status: 202 }); }
|
||||
try { requireManagementOrigin(request); return json({ version: createLibraryUpload(await readGameJson(request, 32 * 1024 * 1024)) }, { status: 202 }); }
|
||||
catch (cause) { return gameFileError(cause); }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { browserUploadStatus } from '$lib/server/game-files/index.js';
|
||||
import { gameFileError } from '$lib/server/game-files/http.js';
|
||||
|
||||
export async function GET({ params }: { params: { id: string } }) {
|
||||
try { return json(await browserUploadStatus(params.id), { headers: { 'Cache-Control': 'no-store' } }); }
|
||||
catch (cause) { return gameFileError(cause); }
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { requestBrowserPublication } from '$lib/server/game-files/index.js';
|
||||
import { gameFileError } from '$lib/server/game-files/http.js';
|
||||
import { requireManagementOrigin } from '$lib/server/hosts/http.js';
|
||||
|
||||
export async function POST({ request, params }: { request: Request; params: { id: string } }) {
|
||||
try {
|
||||
requireManagementOrigin(request);
|
||||
return json(await requestBrowserPublication(params.id), { status: 202 });
|
||||
} catch (cause) { return gameFileError(cause); }
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
import { uploadBrowserFile } from '$lib/server/game-files/index.js';
|
||||
import { gameFileError } from '$lib/server/game-files/http.js';
|
||||
import { requireManagementOrigin } from '$lib/server/hosts/http.js';
|
||||
|
||||
function integerHeader(request: Request, name: string): number {
|
||||
const value = request.headers.get(name);
|
||||
if (value === null || !/^\d+$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error(`Provide a valid ${name} header.`);
|
||||
return Number(value);
|
||||
}
|
||||
export async function PUT({ request, params }: { request: Request; params: { id: string; path: string } }) {
|
||||
try {
|
||||
requireManagementOrigin(request);
|
||||
const chunk = { offset: integerHeader(request, 'X-Upload-Offset'), size: integerHeader(request, 'X-Upload-Length'), sha256: request.headers.get('X-Content-SHA256') ?? '' };
|
||||
return json(await uploadBrowserFile(params.id, params.path, chunk, request.body));
|
||||
} catch (cause) { return gameFileError(cause); }
|
||||
}
|
||||
@@ -1,14 +1,23 @@
|
||||
import { error, json } from '@sveltejs/kit';
|
||||
import { getHost, listHostOperations, updateHost } from '$lib/server/hosts/index.js';
|
||||
import { deleteHost, getHost, hostDeletionStatus, HostDeletionError, listHostOperations, updateHost } from '$lib/server/hosts/index.js';
|
||||
import { hostApiError, objectBody, requireManagementOrigin } from '$lib/server/hosts/http.js';
|
||||
type Event = { request: Request; params: { id: string } };
|
||||
export function GET({ request, params }: Event) {
|
||||
requireManagementOrigin(request); const host = getHost(params.id); if (!host) error(404, 'Host not found.');
|
||||
// Payloads and results can contain RCON keys or staged credentials. History
|
||||
// exposes metadata only; agent APIs remain the only command delivery path.
|
||||
return json({ host, operations: listHostOperations(host.id).map(({ payload: _payload, result: _result, ...operation }) => operation) });
|
||||
return json({ host, deletion: hostDeletionStatus(host.id), operations: listHostOperations(host.id).map(({ payload: _payload, result: _result, ...operation }) => operation) });
|
||||
}
|
||||
export async function PATCH({ request, params }: Event) {
|
||||
requireManagementOrigin(request);
|
||||
try { return json({ host: updateHost(params.id, await objectBody(request)) }); } catch (cause) { return hostApiError(cause); }
|
||||
}
|
||||
|
||||
export function DELETE({ request, params }: Event) {
|
||||
requireManagementOrigin(request);
|
||||
try { const result = deleteHost(params.id); return json(result, { status: result.deleted ? 200 : 202 }); }
|
||||
catch (cause) {
|
||||
if (cause instanceof HostDeletionError) return json({ error: cause.message, blockers: cause.blockers }, { status: cause.status });
|
||||
return hostApiError(cause);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import { onMount, tick } from 'svelte';
|
||||
import { page } from '$app/stores';
|
||||
import { beforeNavigate } from '$app/navigation';
|
||||
import type { GameLibrary, GameLibraryVersion } from '$lib/game-files.js';
|
||||
import { formatGameBytes } from '$lib/game-files.js';
|
||||
import type { Host } from '$lib/hosts.js';
|
||||
import { selectGameFolder, type SelectedGameFolder } from '$ui/game-folder.js';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
|
||||
import Button from '$ui/components/primitives/Button.svelte';
|
||||
@@ -11,110 +13,265 @@
|
||||
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
|
||||
|
||||
type OverviewLibrary = Omit<GameLibrary, 'versions'> & { versions: Omit<GameLibraryVersion, 'files'>[] };
|
||||
let libraries = $state<OverviewLibrary[]>([]);
|
||||
let hosts = $state<Host[]>([]);
|
||||
let loaded = $state(false);
|
||||
let error = $state(''); let notice = $state(''); let busy = $state(false);
|
||||
let showImport = $state(false); let game = $state<'t5' | 't6'>('t5');
|
||||
let name = $state(''); let version = $state(''); let description = $state(''); let dlc = $state(''); let hostId = $state('');
|
||||
let filterGame = $state('all');
|
||||
let targets = $state<Record<string, string>>({});
|
||||
interface UploadFile { path: string; size: number; lastModified: number; receivedBytes: number; complete: boolean }
|
||||
interface UploadStatus { status: 'pending' | 'verifying' | 'published' | 'failed'; chunkSize: number; files: UploadFile[]; error?: string | null; version: { id: string; state: string } }
|
||||
type Phase = 'idle' | 'preparing' | 'uploading' | 'paused' | 'verifying' | 'published';
|
||||
let libraries = $state<OverviewLibrary[]>([]); let hosts = $state<Host[]>([]);
|
||||
let loaded = $state(false); let error = $state(''); let notice = $state(''); let busy = $state(false);
|
||||
let showUpload = $state(false); let game = $state<'t5' | 't6'>('t5');
|
||||
let name = $state(''); let version = $state(''); let description = $state('');
|
||||
let filterGame = $state('all'); let targets = $state<Record<string, string>>({});
|
||||
let folder = $state.raw<SelectedGameFolder | null>(null);
|
||||
let uploadStatus = $state.raw<UploadStatus | null>(null);
|
||||
let uploadId = $state(''); let uploadBusy = $state(false); let uploadError = $state('');
|
||||
let phase = $state<Phase>('idle'); let currentPath = $state(''); let pauseRequested = $state(false);
|
||||
let acceptedBytes = $state(0); let sendingBytes = $state(0); let completedFiles = $state(0);
|
||||
let folderInput = $state<HTMLInputElement>();
|
||||
let activeRequest: XMLHttpRequest | null = null; let disposed = false;
|
||||
const canManage = $derived($page.data.user?.role === 'admin');
|
||||
const onlineHosts = $derived(hosts.filter(host => host.connected && host.credentialActive && host.mode !== 'disabled'));
|
||||
const matching = $derived(libraries.filter(library => filterGame === 'all' || library.game === filterGame));
|
||||
const hostName = (id: string) => hosts.find(host => host.id === id)?.name ?? id;
|
||||
const totalBytes = $derived(folder?.size ?? uploadStatus?.files.reduce((total, file) => total + file.size, 0) ?? 0);
|
||||
const totalFiles = $derived(folder?.files.length ?? uploadStatus?.files.length ?? 0);
|
||||
const progressBytes = $derived(Math.min(totalBytes, acceptedBytes + sendingBytes));
|
||||
const transferring = $derived(uploadBusy && (phase === 'preparing' || phase === 'uploading'));
|
||||
const hostName = (id: string | null) => id ? hosts.find(host => host.id === id)?.name ?? id : 'Browser upload';
|
||||
|
||||
async function responseBody(response: Response) {
|
||||
let body;
|
||||
try { body = await response.json(); } catch { throw new Error('The controller returned an unexpected response. Sign in again if your session expired, then resume the upload.'); }
|
||||
if (!response.ok) throw new Error(body.error ?? 'The controller could not complete this request.');
|
||||
return body;
|
||||
}
|
||||
async function load() {
|
||||
try {
|
||||
const response = await fetch('/api/game-files');
|
||||
const body = await response.json();
|
||||
if (!response.ok) throw new Error(body.error ?? 'Could not load game libraries.');
|
||||
const body = await responseBody(await fetch('/api/game-files'));
|
||||
libraries = body.libraries; hosts = body.hosts; loaded = true;
|
||||
} catch (cause) { error = (cause as Error).message; }
|
||||
}
|
||||
async function act(url: string, payload: Record<string, unknown>, message: string) {
|
||||
async function download(versionId: string) {
|
||||
busy = true; error = ''; notice = '';
|
||||
try {
|
||||
const response = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) });
|
||||
const result = await response.json();
|
||||
if (!response.ok) throw new Error(result.error ?? 'Could not start this operation.');
|
||||
notice = message; await load(); return true;
|
||||
} catch (cause) { error = (cause as Error).message; return false; }
|
||||
finally { busy = false; }
|
||||
await responseBody(await fetch(`/api/game-files/${versionId}/download`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ hostId: targets[versionId] }) }));
|
||||
notice = 'Library download queued. Progress will update below.'; await load();
|
||||
} catch (cause) { error = (cause as Error).message; } finally { busy = false; }
|
||||
}
|
||||
async function startImport(event: SubmitEvent) {
|
||||
event.preventDefault();
|
||||
if (await act('/api/game-files', { game, name, version, description, dlc: dlc.split(',').map(value => value.trim()).filter(Boolean), hostId }, 'Import queued. The source host will scan and upload files, then verify and publish this version.')) showImport = false;
|
||||
function resetUpload() {
|
||||
folder = null; uploadStatus = null; uploadId = ''; phase = 'idle'; uploadError = ''; currentPath = '';
|
||||
acceptedBytes = 0; sendingBytes = 0; completedFiles = 0; pauseRequested = false;
|
||||
if (folderInput) folderInput.value = '';
|
||||
}
|
||||
function newVersion(library: OverviewLibrary) {
|
||||
showImport = true; game = library.game; name = library.name; description = library.description;
|
||||
dlc = library.versions[0]?.dlc.join(', ') ?? ''; version = '';
|
||||
document.getElementById('library-import')?.scrollIntoView({ behavior: 'smooth' });
|
||||
async function newUpload(library?: OverviewLibrary) {
|
||||
resetUpload(); showUpload = true; game = library?.game ?? 't5'; name = library?.name ?? '';
|
||||
description = library?.description ?? ''; version = '';
|
||||
await tick(); document.getElementById('library-upload')?.scrollIntoView({ behavior: 'smooth' });
|
||||
}
|
||||
onMount(() => pollVisible(load, 5000));
|
||||
function chooseFolder(event: Event) {
|
||||
uploadError = ''; folder = null;
|
||||
const input = event.currentTarget as HTMLInputElement;
|
||||
if (!input.files?.length) return;
|
||||
try {
|
||||
const selected = selectGameFolder(Array.from(input.files));
|
||||
if (uploadStatus) checkSelection(selected, uploadStatus);
|
||||
folder = selected;
|
||||
if (!name && !uploadId) name = game === 't5' ? 'Black Ops I game files' : 'Black Ops II game files';
|
||||
} catch (cause) { uploadError = (cause as Error).message; input.value = ''; }
|
||||
}
|
||||
function checkSelection(selected: SelectedGameFolder, status: UploadStatus) {
|
||||
const originals = new Map(status.files.map(file => [file.path, file]));
|
||||
if (selected.files.length !== originals.size || selected.files.some(({ path, file }) => {
|
||||
const original = originals.get(path); return !original || original.size !== file.size || original.lastModified !== file.lastModified;
|
||||
})) throw new Error('This folder does not match the original upload. Select the same unchanged folder, including its original file names, sizes, and modification dates. Upload changed game files as a new version.');
|
||||
}
|
||||
function applyStatus(status: UploadStatus) {
|
||||
uploadStatus = status;
|
||||
acceptedBytes = status.files.reduce((total, file) => total + file.receivedBytes, 0);
|
||||
completedFiles = status.files.filter(file => file.complete).length; sendingBytes = 0;
|
||||
}
|
||||
async function getStatus(): Promise<UploadStatus> {
|
||||
const status = await responseBody(await fetch(`/api/game-files/${uploadId}/upload`)) as UploadStatus;
|
||||
applyStatus(status); return status;
|
||||
}
|
||||
async function followVerification(initial: UploadStatus) {
|
||||
let status = initial; phase = 'verifying'; currentPath = '';
|
||||
while (!disposed && status.status === 'verifying') {
|
||||
await new Promise(resolve => setTimeout(resolve, 1500));
|
||||
if (disposed) return;
|
||||
status = await getStatus();
|
||||
}
|
||||
if (disposed) return;
|
||||
if (status.status !== 'published') throw new Error(status.error || 'Verification did not finish. Retry this upload to continue.');
|
||||
phase = 'published'; folder = null; notice = `${name || 'Game files'} ${version} published. You can download this version to a host below.`;
|
||||
if (folderInput) folderInput.value = '';
|
||||
await load();
|
||||
}
|
||||
async function resumeUpload(library: OverviewLibrary, item: OverviewLibrary['versions'][number]) {
|
||||
resetUpload(); showUpload = true; uploadId = item.id; game = library.game; name = library.name; version = item.version; description = item.description;
|
||||
uploadBusy = true;
|
||||
try {
|
||||
const status = await getStatus();
|
||||
if (status.status === 'verifying' || status.status === 'published') await followVerification(status);
|
||||
else { phase = 'paused'; uploadError = status.error ?? ''; }
|
||||
} catch (cause) { uploadError = (cause as Error).message; phase = 'paused'; }
|
||||
finally { uploadBusy = false; }
|
||||
await tick(); document.getElementById('library-upload')?.scrollIntoView({ behavior: 'smooth' });
|
||||
}
|
||||
function pauseUpload() { pauseRequested = true; activeRequest?.abort(); }
|
||||
async function sendChunk(path: string, file: File, offset: number, length: number): Promise<number> {
|
||||
const chunk = file.slice(offset, offset + length);
|
||||
const hash = await crypto.subtle.digest('SHA-256', await chunk.arrayBuffer());
|
||||
if (pauseRequested || disposed) throw new Error('Upload paused.');
|
||||
const sha256 = [...new Uint8Array(hash)].map(value => value.toString(16).padStart(2, '0')).join('');
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = new XMLHttpRequest(); activeRequest = request;
|
||||
request.open('PUT', `/api/game-files/${uploadId}/upload/files/${path.split('/').map(encodeURIComponent).join('/')}`);
|
||||
request.setRequestHeader('Content-Type', 'application/octet-stream');
|
||||
request.setRequestHeader('X-Upload-Offset', String(offset)); request.setRequestHeader('X-Upload-Length', String(length));
|
||||
request.setRequestHeader('X-Content-SHA256', sha256); request.timeout = 300_000;
|
||||
request.upload.onprogress = event => { sendingBytes = Math.min(length, event.loaded); };
|
||||
request.onload = () => {
|
||||
activeRequest = null;
|
||||
try {
|
||||
const body = JSON.parse(request.responseText);
|
||||
if (request.status < 200 || request.status >= 300) throw new Error(body.error ?? `Upload failed (${request.status}).`);
|
||||
if (body.receivedBytes !== offset + length) throw new Error('The upload position changed. Resume to check the saved progress.');
|
||||
resolve(body.receivedBytes);
|
||||
} catch (cause) { reject(cause); }
|
||||
};
|
||||
request.onerror = () => { activeRequest = null; reject(new Error('The connection was interrupted. Resume to continue from the last saved chunk.')); };
|
||||
request.ontimeout = () => { activeRequest = null; reject(new Error('This chunk timed out. Resume to check saved progress and retry.')); };
|
||||
request.onabort = () => { activeRequest = null; reject(new Error('Upload paused.')); };
|
||||
request.send(chunk);
|
||||
});
|
||||
}
|
||||
async function startUpload(event: SubmitEvent) {
|
||||
event.preventDefault(); if (uploadBusy || !folder) return;
|
||||
if (!globalThis.crypto?.subtle) { uploadError = 'Folder uploads require a secure connection. Open this controller over HTTPS or localhost.'; return; }
|
||||
uploadBusy = true; pauseRequested = false; uploadError = ''; notice = ''; phase = 'preparing';
|
||||
try {
|
||||
if (!uploadId) {
|
||||
const result = await responseBody(await fetch('/api/game-files', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ game, name, version, description, files: folder.files.map(({ path, file }) => ({ path, size: file.size, lastModified: file.lastModified })) }) }));
|
||||
uploadId = result.version.id;
|
||||
}
|
||||
const status = await getStatus();
|
||||
if (status.status === 'verifying' || status.status === 'published') { await followVerification(status); return; }
|
||||
checkSelection(folder, status);
|
||||
const localFiles = new Map(folder.files.map(item => [item.path, item.file]));
|
||||
const chunkSize = Math.min(8_388_608, Math.max(1, status.chunkSize));
|
||||
phase = 'uploading';
|
||||
for (const remote of status.files) {
|
||||
if (pauseRequested || disposed) break;
|
||||
if (remote.complete) continue;
|
||||
currentPath = remote.path;
|
||||
const file = localFiles.get(remote.path)!;
|
||||
do {
|
||||
if (pauseRequested || disposed) break;
|
||||
const length = Math.min(chunkSize, remote.size - remote.receivedBytes);
|
||||
remote.receivedBytes = await sendChunk(remote.path, file, remote.receivedBytes, length);
|
||||
acceptedBytes += length; sendingBytes = 0;
|
||||
if (remote.receivedBytes === remote.size) { remote.complete = true; completedFiles++; }
|
||||
} while (!remote.complete);
|
||||
}
|
||||
if (pauseRequested || disposed) { phase = 'paused'; return; }
|
||||
phase = 'verifying'; currentPath = '';
|
||||
const verification = await responseBody(await fetch(`/api/game-files/${uploadId}/upload/complete`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' })) as UploadStatus;
|
||||
applyStatus(verification); await followVerification(verification);
|
||||
} catch (cause) {
|
||||
if (!disposed) { uploadError = pauseRequested ? '' : (cause as Error).message; phase = 'paused'; }
|
||||
} finally { uploadBusy = false; sendingBytes = 0; if (!disposed) void load(); }
|
||||
}
|
||||
beforeNavigate(navigation => {
|
||||
if (!transferring) return;
|
||||
if (navigation.type === 'leave' || !window.confirm('This upload is still transferring files. Leave this page? You can resume later by selecting the original unchanged folder.')) navigation.cancel();
|
||||
});
|
||||
onMount(() => {
|
||||
const stop = pollVisible(load, 5000);
|
||||
return () => { disposed = true; pauseRequested = true; activeRequest?.abort(); stop(); };
|
||||
});
|
||||
</script>
|
||||
|
||||
<svelte:head><title>Game Files | Electric Cherry</title></svelte:head>
|
||||
<div class="page">
|
||||
<SectionHeader title="Game Files" subtitle="Import immutable base-game versions and distribute verified copies to your hosts.">
|
||||
{#snippet actions()}{#if canManage}<Button onclick={() => showImport = !showImport}>{showImport ? 'Close import' : 'Import game files'}</Button>{/if}{/snippet}
|
||||
<SectionHeader title="Game Files" subtitle="Upload your T5 or T6 game folder, then distribute verified versions to your hosts.">
|
||||
{#snippet actions()}{#if canManage}<Button disabled={uploadBusy} onclick={() => newUpload()}>Upload game folder</Button>{/if}{/snippet}
|
||||
</SectionHeader>
|
||||
{#if error}<p class="ui-error" role="alert">{error}</p>{/if}
|
||||
{#if notice}<p role="status">{notice}</p>{/if}
|
||||
{#if !canManage}<p class="muted">Administrators import game files and distribute verified copies to hosts.</p>{/if}
|
||||
{#if canManage && showImport}
|
||||
<section class="card" id="library-import">
|
||||
<h2>Import an existing game volume</h2>
|
||||
<p>The selected host scans its {game}_game_files Docker volume, calculates SHA256 checksums, and streams a copy into controller storage. Publication requires every file to pass verification.</p>
|
||||
<form onsubmit={startImport}>
|
||||
<div class="fields">
|
||||
{#if !canManage}<p class="muted">Administrators upload game files and distribute verified copies to hosts.</p>{/if}
|
||||
{#if canManage && showUpload}
|
||||
<section class="card" id="library-upload">
|
||||
<div class="heading"><h2>{uploadId ? phase === 'published' ? 'Version published' : 'Continue folder upload' : 'Upload a game folder'}</h2><Button size="sm" variant="ghost" disabled={uploadBusy} onclick={() => showUpload = false}>Close</Button></div>
|
||||
<p>Select your complete T5 or T6 game folder, including its installed DLC. The files upload directly to this controller; a game host is only needed when you are ready to run servers.</p>
|
||||
{#if uploadError}<p class="ui-error" role="alert">{uploadError}</p>{/if}
|
||||
<form onsubmit={startUpload}>
|
||||
<fieldset disabled={uploadBusy || !!uploadId} class="fields">
|
||||
<label>Game<select bind:value={game}><option value="t5">Black Ops I (T5)</option><option value="t6">Black Ops II (T6)</option></select></label>
|
||||
<label>Source host<select required bind:value={hostId}><option value="">Select an online host</option>{#each onlineHosts as host}<option value={host.id}>{host.name}</option>{/each}</select></label>
|
||||
<label>Library name<input required maxlength="160" bind:value={name} placeholder="Black Ops II — Complete Server Files" /></label>
|
||||
<label>New version label<input required maxlength="120" bind:value={version} placeholder="2026-10-03" /></label>
|
||||
<label class="wide">Description<textarea maxlength="4000" bind:value={description} rows="2"></textarea></label>
|
||||
<label class="wide">Content / DLC (comma separated)<input bind:value={dlc} placeholder="Base game, Nuketown, Die Rise, Mob of the Dead, Buried, Origins" /></label>
|
||||
<label>Library name<input required maxlength="160" bind:value={name} placeholder="Black Ops II game files" /></label>
|
||||
<label>Version label<input required maxlength="120" bind:value={version} placeholder="2026-10-04" /></label>
|
||||
<label class="wide">Description <span class="muted">(optional)</span><textarea maxlength="4000" bind:value={description} rows="2"></textarea></label>
|
||||
</fieldset>
|
||||
{#if phase !== 'verifying' && phase !== 'published'}
|
||||
<label class="folder-picker">{uploadId ? 'Select the original unchanged folder' : 'Game folder'}<input bind:this={folderInput} type="file" webkitdirectory multiple onchange={chooseFolder} disabled={uploadBusy} aria-describedby="folder-help" /></label>
|
||||
<p class="muted" id="folder-help">{uploadId ? 'To resume, file names, sizes, and modification dates must match the original upload.' : 'Choose the game directory itself. Its top-level folder name is removed once; all files and subfolders inside it keep their paths.'}</p>
|
||||
{/if}
|
||||
{#if folder}<div class="folder-summary"><strong>{folder.name}</strong><span>{folder.files.length.toLocaleString()} files · {formatGameBytes(folder.size)}</span><details><summary>Preview folder paths</summary><ul class="path-list">{#each folder.files.slice(0, 8) as file}<li>{file.path}</li>{/each}</ul>{#if folder.files.length > 8}<p class="muted">And {(folder.files.length - 8).toLocaleString()} more files.</p>{/if}</details></div>{/if}
|
||||
{#if phase !== 'idle'}<div class="upload-progress" aria-live="polite">
|
||||
<div class="heading"><strong>{phase === 'preparing' ? 'Preparing upload…' : phase === 'uploading' ? 'Uploading game files' : phase === 'verifying' ? 'Verifying and publishing…' : phase === 'published' ? 'Published and ready to distribute' : 'Upload paused'}</strong><span>{completedFiles.toLocaleString()} / {totalFiles.toLocaleString()} files</span></div>
|
||||
<progress max={totalBytes || Math.max(1, totalFiles)} value={totalBytes ? progressBytes : completedFiles} aria-label="Game folder upload progress"></progress>
|
||||
<p>{formatGameBytes(progressBytes)} / {formatGameBytes(totalBytes)} saved or transferring</p>
|
||||
{#if currentPath}<p class="current-file">{currentPath}</p>{/if}
|
||||
{#if phase === 'verifying'}<p class="muted">The controller checks every file and publishes this version when verification finishes. Verification continues if you leave this page.</p>{:else if phase === 'paused'}<p class="muted">Saved chunks are kept on the controller. Resume this upload to check the saved progress and retry.</p>{/if}
|
||||
</div>{/if}
|
||||
<div class="upload-actions">
|
||||
{#if transferring}<Button onclick={pauseUpload} disabled={pauseRequested}>{pauseRequested ? 'Pausing…' : 'Pause upload'}</Button>
|
||||
{:else if phase !== 'verifying' && phase !== 'published'}<Button type="submit" variant="primary" disabled={uploadBusy || !folder}>{uploadId ? 'Resume upload' : 'Upload and publish'}</Button>{/if}
|
||||
{#if phase === 'idle' || phase === 'uploading'}<span class="muted">Keep this page open while files transfer. Interrupted uploads can be resumed below.</span>{/if}
|
||||
</div>
|
||||
<Button type="submit" disabled={busy || !onlineHosts.length}>Start import</Button>
|
||||
{#if !onlineHosts.length}<p>Enroll an online source host in <a href="/hosts">Hosts</a> before importing.</p>{/if}
|
||||
</form>
|
||||
</section>
|
||||
{/if}
|
||||
<div class="filters"><label>Game<select bind:value={filterGame}><option value="all">All games</option><option value="t5">Black Ops I</option><option value="t6">Black Ops II</option></select></label><span class="muted">{libraries.length} libraries · Base-game files and runtime images are versioned separately.</span></div>
|
||||
<div class="filters"><label>Game<select bind:value={filterGame}><option value="all">All games</option><option value="t5">Black Ops I</option><option value="t6">Black Ops II</option></select></label><span class="muted">{libraries.length} libraries · Game folders and runtime images are versioned separately.</span></div>
|
||||
{#each matching as library (library.id)}
|
||||
<section class="card">
|
||||
<div class="heading"><div><h2>{library.name}</h2><p class="muted">{library.game.toUpperCase()} · {library.description}</p></div>{#if canManage}<Button size="sm" onclick={() => newVersion(library)}>Import new version</Button>{/if}</div>
|
||||
<div class="heading"><div><h2>{library.name}</h2><p class="muted">{library.game.toUpperCase()} · {library.description}</p></div>{#if canManage}<Button size="sm" disabled={uploadBusy} onclick={() => newUpload(library)}>Upload new version</Button>{/if}</div>
|
||||
{#each library.versions as item (item.id)}
|
||||
{@const browserUpload = !item.sourceHostId}
|
||||
<article class="version">
|
||||
<div class="heading"><h3>{item.version}</h3><Badge variant={item.state === 'failed' ? 'danger' : item.state === 'ready' ? 'online' : 'info'}>{item.state === 'ready' ? 'Published' : item.state === 'importing' ? 'Importing' : 'Failed'}</Badge></div>
|
||||
<div class="heading"><h3>{item.version}</h3><Badge variant={item.state === 'failed' ? 'danger' : item.state === 'ready' ? 'online' : 'info'}>{item.state === 'ready' ? 'Published' : item.state === 'importing' ? browserUpload ? item.id === uploadId && phase === 'verifying' ? 'Verifying' : 'Uploading' : 'Importing' : 'Failed'}</Badge></div>
|
||||
<p>{formatGameBytes(item.sizeBytes)} · {item.fileCount.toLocaleString()} files · Created {new Date(item.createdAt).toLocaleString()}</p>
|
||||
{#if item.dlc.length}<p>Content: {item.dlc.join(', ')}</p>{/if}
|
||||
{#if item.error}<p class="ui-error">{item.error}</p>{/if}
|
||||
<details><summary>Version details</summary><dl><dt>Immutable version ID</dt><dd>{item.id}</dd><dt>Manifest SHA256</dt><dd>{item.manifestSha256 ?? 'Waiting for source scan'}</dd><dt>Controller storage</dt><dd>{item.storageLocation}</dd><dt>Imported from</dt><dd>{hostName(item.sourceHostId)} / {item.sourceVolume}</dd></dl><a href={`/api/game-files/${item.id}`} target="_blank" rel="noreferrer">View complete file manifest</a></details>
|
||||
<details><summary>Version details</summary><dl><dt>Immutable version ID</dt><dd>{item.id}</dd><dt>Manifest SHA256</dt><dd>{item.manifestSha256 ?? 'Available after verification'}</dd><dt>Uploaded from</dt><dd>{browserUpload ? 'Game folder in browser' : `${hostName(item.sourceHostId)} / ${item.sourceVolume}`}</dd></dl>{#if item.state === 'ready'}<a href={`/api/game-files/${item.id}`} target="_blank" rel="noreferrer">View complete file manifest</a>{/if}</details>
|
||||
{#if item.state === 'ready'}
|
||||
{#if canManage}<div class="download"><label>Download to host<select bind:value={targets[item.id]}><option value="">Select host</option>{#each onlineHosts as host}<option value={host.id}>{host.name} · {formatGameBytes(Number(host.inventory.storageAvailable ?? 0))} free</option>{/each}</select></label><Button size="sm" disabled={busy || !targets[item.id]} onclick={() => act(`/api/game-files/${item.id}/download`, { hostId: targets[item.id] }, 'Library download queued. Progress will update below.')}>Download / verify</Button></div>{/if}
|
||||
{#if canManage}<div class="download"><label>Download to host<select bind:value={targets[item.id]}><option value="">Select host</option>{#each onlineHosts as host}<option value={host.id}>{host.name} · {formatGameBytes(Number(host.inventory.storageAvailable ?? 0))} free</option>{/each}</select></label><Button size="sm" disabled={busy || !targets[item.id]} onclick={() => download(item.id)}>Download / verify</Button></div>{#if !onlineHosts.length}<p class="muted">This version is stored and ready. <a href="/hosts">Connect a host</a> when you want to deploy game servers.</p>{/if}{/if}
|
||||
<div class="table-wrap"><table><thead><tr><th>Host</th><th>Cache state</th><th>Progress</th><th>Detail</th></tr></thead><tbody>
|
||||
{#each item.caches as cache}<tr><td><a href={`/hosts/${cache.hostId}`}>{hostName(cache.hostId)}</a></td><td>{cache.status}{!hosts.find(host => host.id === cache.hostId)?.connected ? ' · offline' : ''}</td><td>{#if cache.status === 'downloading' || cache.status === 'verifying'}<progress max={cache.bytesTotal || 1} value={cache.bytesDownloaded}></progress><small>{formatGameBytes(cache.bytesDownloaded)} / {formatGameBytes(cache.bytesTotal)}</small>{:else if cache.status === 'ready' || cache.status === 'outdated'}Verified{cache.status === 'outdated' ? ' · older version' : ''}{:else}—{/if}</td><td>{cache.error ?? ''}</td></tr>{:else}<tr><td colspan="4">Enroll a host to cache this version.</td></tr>{/each}
|
||||
{#each item.caches as cache}<tr><td><a href={`/hosts/${cache.hostId}`}>{hostName(cache.hostId)}</a></td><td>{cache.status}{!hosts.find(host => host.id === cache.hostId)?.connected ? ' · offline' : ''}</td><td>{#if cache.status === 'downloading' || cache.status === 'verifying'}<progress max={cache.bytesTotal || 1} value={cache.bytesDownloaded} aria-label="Host download progress"></progress><small>{formatGameBytes(cache.bytesDownloaded)} / {formatGameBytes(cache.bytesTotal)}</small>{:else if cache.status === 'ready' || cache.status === 'outdated'}Verified{cache.status === 'outdated' ? ' · older version' : ''}{:else}—{/if}</td><td>{cache.error ?? ''}</td></tr>{:else}<tr><td colspan="4">No host copies yet.</td></tr>{/each}
|
||||
</tbody></table></div>
|
||||
{:else if item.state === 'importing'}<p class="muted">The agent operation continues independently of this page. An interrupted import is reconciled when the host reconnects.</p>{/if}
|
||||
{:else if browserUpload}<p class="muted">Resume with the original unchanged folder. Published versions can be downloaded to any enrolled host.</p>{#if canManage}<div><Button size="sm" disabled={uploadBusy} onclick={() => resumeUpload(library, item)}>Resume upload</Button></div>{/if}
|
||||
{:else if item.state === 'importing'}<p class="muted">The existing host import continues independently. It is reconciled when the host reconnects.</p>{/if}
|
||||
</article>
|
||||
{/each}
|
||||
</section>
|
||||
{:else}{#if loaded}<EmptyState title="No game libraries yet" description="Administrators can import T5 or T6 game files from an enrolled host. Each published version can be shared by many instances." />{:else}<p role="status">Loading game libraries…</p>{/if}{/each}
|
||||
{:else}{#if loaded}<EmptyState title="No game libraries yet" description="Upload a complete T5 or T6 game folder to get started. You can add a host after the files are published." />{:else}<p role="status">Loading game libraries…</p>{/if}{/each}
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.page, .card, .version, form { display: grid; gap: 20px; }
|
||||
.card { padding: 24px; border: 1px solid var(--border-subtle); background: var(--surface); border-radius: var(--radius-md); }
|
||||
.version { padding-top: 20px; border-top: 1px solid var(--border-subtle); }
|
||||
.heading, .filters, .download { display: flex; gap: 16px; align-items: center; justify-content: space-between; flex-wrap: wrap; }
|
||||
.download { justify-content: flex-start; align-items: end; }
|
||||
.fields { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 16px; }
|
||||
.wide { grid-column: 1 / -1; } label { display: grid; gap: 6px; font-size: var(--text-sm); }
|
||||
input, select, textarea { width: 100%; min-width: 0; box-sizing: border-box; padding: 10px; background: var(--surface); color: inherit; border: 1px solid var(--border-strong); border-radius: var(--radius-sm); }
|
||||
h2, h3, p { margin: 0; } h2 { font-size: var(--text-md); } h3 { font-size: var(--text-sm); }
|
||||
p, td, th, details { font-size: var(--text-sm); line-height: 1.6; } .muted, small { color: var(--text-secondary); }
|
||||
small { display: block; } a { color: var(--accent-primary-hover); } summary { cursor: pointer; }
|
||||
dl { display: grid; grid-template-columns: auto 1fr; gap: 6px 16px; } dd { margin: 0; overflow-wrap: anywhere; }
|
||||
.table-wrap { overflow-x: auto; } table { width: 100%; border-collapse: collapse; text-align: left; } th, td { padding: 10px; border-bottom: 1px solid var(--border-subtle); }
|
||||
@media(max-width: 640px) { .fields { grid-template-columns: 1fr; } .card { padding: 16px; } dl { grid-template-columns: 1fr; } }
|
||||
.page, .card, .version, form { display:grid; gap:20px; }
|
||||
.card { padding:24px; min-width:0; border:1px solid var(--border-subtle); background:var(--surface); border-radius:var(--radius-md); }
|
||||
.version { padding-top:20px; border-top:1px solid var(--border-subtle); }
|
||||
.heading, .filters, .download, .upload-actions { display:flex; gap:16px; align-items:center; justify-content:space-between; flex-wrap:wrap; }
|
||||
.download, .upload-actions { justify-content:flex-start; align-items:end; }
|
||||
.fields { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:16px; border:0; padding:0; margin:0; min-width:0; }
|
||||
.wide { grid-column:1/-1; } label { display:grid; gap:6px; font-size:var(--text-sm); }
|
||||
input, select, textarea { width:100%; min-width:0; box-sizing:border-box; padding:10px; background:var(--surface); color:inherit; border:1px solid var(--border-strong); border-radius:var(--radius-sm); }
|
||||
input:disabled, select:disabled, textarea:disabled { opacity:.7; } input[type='file'] { padding:16px; cursor:pointer; }
|
||||
.folder-summary,.upload-progress { padding:20px; display:grid; gap:12px; background:var(--surface-sunken); border:1px solid var(--border-subtle); border-radius:var(--radius-md); min-width:0; }
|
||||
.folder-summary>span,.upload-progress span,.upload-actions>span { color:var(--text-secondary); font-size:var(--text-sm); }
|
||||
.path-list { padding-left:20px; font-family:var(--font-mono); font-size:var(--text-xs); overflow-wrap:anywhere; }
|
||||
.current-file { font-family:var(--font-mono); font-size:var(--text-xs); overflow-wrap:anywhere; } progress { width:100%; accent-color:var(--accent-primary); }
|
||||
h2,h3,p { margin:0; } h2 { font-size:var(--text-md); } h3 { font-size:var(--text-sm); }
|
||||
p,td,th,details { font-size:var(--text-sm); line-height:1.6; } .muted,small { color:var(--text-secondary); }
|
||||
small { display:block; } a { color:var(--accent-primary-hover); } summary { cursor:pointer; }
|
||||
dl { display:grid; grid-template-columns:auto 1fr; gap:6px 16px; } dd { margin:0; overflow-wrap:anywhere; }
|
||||
.table-wrap { overflow-x:auto; } table { width:100%; border-collapse:collapse; text-align:left; } th,td { padding:10px; border-bottom:1px solid var(--border-subtle); }
|
||||
@media(max-width:640px) { .fields { grid-template-columns:1fr; } .card,.folder-summary,.upload-progress { padding:16px; } dl { grid-template-columns:1fr; } .heading>div { min-width:0; } }
|
||||
</style>
|
||||
|
||||
@@ -2,16 +2,27 @@
|
||||
import { onMount } from 'svelte';
|
||||
import { page } from '$app/stores';
|
||||
import type { Host } from '$lib/hosts.js';
|
||||
import { hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
|
||||
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
|
||||
import Button from '$ui/components/primitives/Button.svelte';
|
||||
import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
|
||||
let hosts = $state<Host[]>([]);
|
||||
let loading = $state(true); let busy = $state(false); let error = $state('');
|
||||
let notice = $state(''); let deleteOpen = $state(false); let deletingHost = $state<Host | null>(null);
|
||||
let loadSequence = 0;
|
||||
let name = $state(''); let portMin = $state(28960); let portMax = $state(28980);
|
||||
let enrollment = $state<{ command: string; expiresAt: string; hostId: string } | null>(null);
|
||||
const canManage = $derived($page.data.user?.role === 'admin');
|
||||
async function load() {
|
||||
try { const response = await fetch('/api/hosts'); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); hosts = body.hosts; }
|
||||
catch (cause) { error = (cause as Error).message; } finally { loading = false; }
|
||||
const sequence = ++loadSequence;
|
||||
try { const response = await fetch('/api/hosts'); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); if (sequence === loadSequence) { hosts = body.hosts; error = ''; } }
|
||||
catch (cause) { if (sequence === loadSequence) error = (cause as Error).message; } finally { if (sequence === loadSequence) loading = false; }
|
||||
}
|
||||
async function deleted(result: { deleted: boolean }) {
|
||||
notice = result.deleted ? `${deletingHost?.name ?? 'Host'} deleted.` : `Cleanup requested for ${deletingHost?.name ?? 'host'}. It remains listed until its files and agent are removed.`;
|
||||
if (enrollment?.hostId === deletingHost?.id) enrollment = null;
|
||||
deletingHost = null;
|
||||
await load();
|
||||
}
|
||||
async function add(event: SubmitEvent) {
|
||||
event.preventDefault(); busy = true; error = ''; enrollment = null;
|
||||
@@ -28,6 +39,7 @@
|
||||
<section class="page">
|
||||
<SectionHeader title="Hosts" subtitle="Connect game-server hosts and prepare them for deployment." />
|
||||
{#if error}<p role="alert" class="ui-error">{error}</p>{/if}
|
||||
{#if notice}<p role="status">{notice}</p>{/if}
|
||||
{#if canManage}<section class="panel">
|
||||
<h2>Add host</h2><p>Run the installation command on a Debian 12 or 13 amd64 machine. Each host connects to this controller over HTTPS.</p>
|
||||
<form onsubmit={add}><fieldset disabled={busy}>
|
||||
@@ -39,14 +51,15 @@
|
||||
{#if enrollment}<div class="enrollment" role="status"><h3>Install the agent</h3><p>This single-use command expires {new Date(enrollment.expiresAt).toLocaleString()}. Run it with administrator access on your game host.</p><pre>{enrollment.command}</pre><a href={`/hosts/${enrollment.hostId}`}>Open host setup →</a></div>{/if}
|
||||
</section>{:else}<p>Administrators manage host setup, runtime preparation, and credentials.</p>{/if}
|
||||
{#if loading}<p role="status">Loading hosts…</p>{:else if !hosts.length}<p>No hosts connected yet.{canManage ? ' Add a host to begin.' : ''}</p>{:else}
|
||||
<div class="host-grid">{#each hosts as host}<a class="panel host" href={`/hosts/${host.id}`}><div class="row"><h2>{host.name}</h2><span class:ready={host.readiness === 'ready'} class="state">{host.readiness}</span></div><p>{host.hostname || 'Awaiting enrollment'} · {host.connectionState}</p><p>{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}</p><p>Game ports {host.portMin}–{host.portMax}</p>{#if host.readinessReasons.length}<p class="reason">{host.readinessReasons[0]}</p>{/if}</a>{/each}</div>
|
||||
<div class="host-grid">{#each hosts as host (host.id)}<article class="panel host"><div class="row"><h2><a href={`/hosts/${host.id}`}>{host.name}</a></h2><span class:ready={host.readiness === 'ready'} class="state">{hostReadinessLabel(host.readiness)}</span></div><p>{host.hostname || 'Awaiting enrollment'} · {host.connectionState}</p><p>{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}</p><p>Game ports {host.portMin}–{host.portMax}</p>{#if host.decommission}<p class="reason">{hostDeletionMessage(host)}</p>{#if host.decommission.error}<p class="ui-error">{host.decommission.error}</p>{/if}{:else if host.readinessReasons.length}<p class="reason">{host.readinessReasons[0]}</p>{/if}<div class="host-actions"><Button size="sm" href={`/hosts/${host.id}`}>{host.decommission ? 'View cleanup' : 'Manage host'}</Button>{#if host.readiness === 'needs-game-files'}<Button size="sm" href="/game-files">Choose game files</Button>{/if}{#if canManage && (!host.decommission || host.decommission.retryable)}<Button size="sm" variant="danger" onclick={() => { deletingHost = host; deleteOpen = true; }}>{host.decommission ? 'Retry cleanup' : 'Delete host'}</Button>{/if}</div></article>{/each}</div>
|
||||
{/if}
|
||||
</section>
|
||||
<DeleteHostDialog bind:open={deleteOpen} host={deletingHost} ondeleted={deleted} />
|
||||
|
||||
<style>
|
||||
.page { display:grid; gap:24px; } .panel { border:1px solid var(--border-strong); border-radius:var(--radius-md); background:var(--surface); padding:24px; }
|
||||
h2 { font-size:var(--text-md); margin:0; } h3 { font-size:var(--text-sm); } p { color:var(--text-secondary); line-height:1.6; font-size:var(--text-sm); }
|
||||
fieldset { display:flex; gap:16px; flex-wrap:wrap; align-items:end; border:0; padding:0; margin-top:20px; } label { display:grid; gap:8px; font-size:var(--text-sm); } input { width:100%; max-width:280px; padding:10px; color:var(--text-primary); background:var(--surface-sunken); border:1px solid var(--border-control); border-radius:var(--radius-sm); }
|
||||
pre { white-space:pre-wrap; overflow-wrap:anywhere; padding:16px; background:var(--surface-sunken); font-size:var(--text-sm); } .enrollment { border-top:1px solid var(--border-subtle); margin-top:24px; padding-top:16px; }
|
||||
.host-grid { display:grid; grid-template-columns:repeat(auto-fit,minmax(300px,1fr)); gap:16px; } .host { text-decoration:none; color:inherit; } .host:hover { border-color:var(--accent-primary); } .row { display:flex; justify-content:space-between; gap:16px; } .state { font-size:var(--text-xs); text-transform:capitalize; color:var(--text-secondary); } .ready { color:var(--accent-primary); } .reason { margin-bottom:0; }
|
||||
.host-grid { display:grid; grid-template-columns:repeat(auto-fit,minmax(300px,1fr)); gap:16px; } .host h2 a { color:inherit; text-decoration:none; } .host h2 a:hover { color:var(--accent-primary); } .host-actions { display:flex; flex-wrap:wrap; gap:12px; margin-top:20px; } .row { display:flex; justify-content:space-between; gap:16px; } .state { font-size:var(--text-xs); text-transform:capitalize; color:var(--text-secondary); } .ready { color:var(--accent-primary); } .reason { margin-bottom:0; }
|
||||
</style>
|
||||
|
||||
@@ -1,19 +1,24 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import { page } from '$app/stores';
|
||||
import { goto } from '$app/navigation';
|
||||
import type { Host, HostOperation, HostMode } from '$lib/hosts.js';
|
||||
import { hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
|
||||
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
|
||||
import Button from '$ui/components/primitives/Button.svelte';
|
||||
import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
|
||||
let host = $state<Host | null>(null); let operations = $state<HostOperation[]>([]);
|
||||
let loading = $state(true); let busy = $state(false); let error = $state(''); let notice = $state('');
|
||||
let deleteOpen = $state(false);
|
||||
let name = $state(''); let portMin = $state(28960); let portMax = $state(28980); let bindAddress = $state('0.0.0.0'); let publicAddress = $state(''); let minimumFreeGB = $state(1);
|
||||
let runtimeImage = $state('ghcr.io/ayymoss/plutainer:latest'); let enrollment = $state<{ command: string; expiresAt: string } | null>(null);
|
||||
const endpoint = $derived(`/api/hosts/${$page.params.id}`);
|
||||
const canManage = $derived($page.data.user?.role === 'admin');
|
||||
const cleaning = $derived(!!host?.decommission);
|
||||
function bytes(value: unknown) { return typeof value === 'number' ? `${(value / 1073741824).toFixed(1)} GiB` : 'Unknown'; }
|
||||
async function load(initialize = false) {
|
||||
try {
|
||||
const response = await fetch(endpoint); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations;
|
||||
const response = await fetch(endpoint); const body = await response.json(); if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations;
|
||||
if (initialize && host) { name = host.name; portMin = host.portMin; portMax = host.portMax; bindAddress = host.bindAddress; publicAddress = host.publicAddress; minimumFreeGB = host.minimumFreeBytes / 1073741824; }
|
||||
} catch (cause) { error = (cause as Error).message; } finally { loading = false; }
|
||||
}
|
||||
@@ -28,27 +33,32 @@
|
||||
</script>
|
||||
|
||||
<svelte:head><title>{host?.name ?? 'Host'} · Electric Cherry</title></svelte:head>
|
||||
<section class="page"><a href="/hosts">← All hosts</a><SectionHeader title={host?.name ?? 'Host'} subtitle="Readiness, inventory, credentials, and operation history." />
|
||||
<section class="page"><a href="/hosts">← All hosts</a><SectionHeader title={host?.name ?? 'Host'} subtitle="Readiness, inventory, credentials, and operation history.">
|
||||
{#snippet actions()}{#if canManage && host && (!cleaning || host.decommission?.retryable)}<Button variant="danger" disabled={busy} onclick={() => deleteOpen = true}>{cleaning ? 'Retry cleanup' : 'Delete host'}</Button>{/if}{/snippet}
|
||||
</SectionHeader>
|
||||
{#if error}<p class="ui-error" role="alert">{error}</p>{/if}{#if notice}<p role="status">{notice}</p>{/if}
|
||||
{#if loading}<p role="status">Loading host…</p>{:else if host}
|
||||
<section class="panel"><div class="row"><h2>{host.connectionState} · {host.readiness}</h2><span>Last heartbeat: {host.lastHeartbeat ? new Date(host.lastHeartbeat).toLocaleString() : 'Never'}</span></div>
|
||||
{#if host.decommission}<section class="panel" aria-live="polite"><h2>Host cleanup</h2><p>{hostDeletionMessage(host)}</p>{#if host.decommission.error}<p class="ui-error">{host.decommission.error}</p>{/if}<p>Requested {new Date(host.decommission.requestedAt).toLocaleString()}. This page closes only after the cleanup worker confirms removal.</p>{#if host.decommission.stage === 'cleaned'}<p>The agent is expected to go offline during uninstall. If final confirmation never arrives, inspect the machine; loss of contact alone does not prove completion.</p>{/if}</section>{/if}
|
||||
<section class="panel"><div class="row"><h2>{host.connectionState} · {hostReadinessLabel(host.readiness)}</h2><span>Last heartbeat: {host.lastHeartbeat ? new Date(host.lastHeartbeat).toLocaleString() : 'Never'}</span></div>
|
||||
{#if host.readinessReasons.length}<ul>{#each host.readinessReasons as reason}<li>{reason}</li>{/each}</ul>{:else}<p>Connected and ready. Each deployment also checks its game library, runtime, and ports.</p>{/if}
|
||||
{#if canManage}<div class="actions"><Button disabled={busy || host.mode === 'active'} onclick={() => mode('active')}>Enable</Button><Button disabled={busy || host.mode === 'draining'} onclick={() => mode('draining')}>Drain host</Button><Button disabled={busy || host.mode === 'disabled'} onclick={() => mode('disabled')}>Disable host</Button></div>{/if}
|
||||
{#if host.readiness === 'needs-game-files'}<p><a href="/game-files">Choose a published game-file version and download it to this host →</a></p>{/if}
|
||||
{#if canManage && !cleaning}<div class="actions"><Button disabled={busy || host.mode === 'active'} onclick={() => mode('active')}>Enable</Button><Button disabled={busy || host.mode === 'draining'} onclick={() => mode('draining')}>Drain host</Button><Button disabled={busy || host.mode === 'disabled'} onclick={() => mode('disabled')}>Disable host</Button></div>{/if}
|
||||
<p>Draining prevents new deployments and retires instances as they become safe to remove. Disabled hosts receive no new commands; running game containers continue.</p>
|
||||
</section>
|
||||
<div class="grid"><section class="panel"><h2>Inventory</h2><dl>
|
||||
<dt>Hostname</dt><dd>{host.hostname || 'Pending'}</dd><dt>Agent</dt><dd>{host.inventory.agentVersion ?? 'Unknown'}</dd><dt>Platform</dt><dd>{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}</dd><dt>CPU</dt><dd>{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)</dd><dt>Memory available / total</dt><dd>{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}</dd><dt>Game storage available / total</dt><dd>{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}</dd><dt>Docker</dt><dd>{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}</dd><dt>Current operation</dt><dd>{host.inventory.currentOperation ?? 'None reported'}</dd>
|
||||
</dl></section><section class="panel"><h2>Host configuration</h2>{#if canManage}<form onsubmit={save}><fieldset disabled={busy}>
|
||||
</dl></section><section class="panel"><h2>Host configuration</h2>{#if canManage}<form onsubmit={save}><fieldset disabled={busy || cleaning}>
|
||||
<label>Friendly name<input required maxlength="200" bind:value={name} /></label><div class="fields"><label>First game port<input type="number" min="1" max="65535" required bind:value={portMin} /></label><label>Last game port<input type="number" min={portMin} max="65535" required bind:value={portMax} /></label></div><label>Local bind address<input required bind:value={bindAddress} /></label><label>Public / NAT address<input bind:value={publicAddress} placeholder="Optional public hostname or address" /></label><label>Minimum free game storage (GiB)<input type="number" min="0" step="0.1" required bind:value={minimumFreeGB} /></label><Button type="submit" disabled={busy}>Save settings</Button>
|
||||
</fieldset></form>{:else}<dl><dt>Friendly name</dt><dd>{host.name}</dd><dt>Game ports</dt><dd>{host.portMin}–{host.portMax}</dd><dt>Local bind address</dt><dd>{host.bindAddress}</dd><dt>Public / NAT address</dt><dd>{host.publicAddress || 'Not configured'}</dd><dt>Minimum free storage</dt><dd>{bytes(host.minimumFreeBytes)}</dd></dl><p>Host configuration is managed by administrators.</p>{/if}<p>Public addressing is separate from local port reservations. Configure your firewall and NAT for the game-port range.</p></section></div>
|
||||
<div class="grid"><section class="panel"><h2>Game files and runtime</h2><p><a href="/game-files">Manage game-file libraries →</a></p>{#if host.inventory.libraries?.length}<ul>{#each host.inventory.libraries as library}<li>{library.id}: {library.state}{library.progress !== undefined ? ` (${library.progress}%)` : ''}</li>{/each}</ul>{:else}<p>No cached game libraries reported.</p>{/if}
|
||||
{#if canManage}<form onsubmit={(event) => { event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}><fieldset disabled={busy}><label>Plutainer image<input required bind:value={runtimeImage} /></label><Button type="submit" disabled={busy || !host.connected}>Prepare runtime</Button></fieldset></form>{/if}<p>Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported'}</p>
|
||||
</section><section class="panel"><h2>Agent access</h2><p>Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.</p>{#if canManage}<div class="actions"><Button disabled={busy} onclick={() => action('/enrollment', 'POST')}>Generate installer</Button><Button disabled={busy || !host.credentialActive || !host.connected} onclick={() => action('/credentials', 'POST', undefined, 'Credential rotation queued.')}>Rotate credential</Button><Button disabled={busy || !host.credentialActive} onclick={() => action('/credentials', 'DELETE', undefined, 'Host credentials revoked. Generate a new installer to enroll again.')}>Revoke credential</Button></div>
|
||||
{#if canManage}<form onsubmit={(event) => { event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}><fieldset disabled={busy || cleaning}><label>Plutainer image<input required bind:value={runtimeImage} /></label><Button type="submit" disabled={busy || !host.connected}>Prepare runtime</Button></fieldset></form>{/if}<p>Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported'}</p>
|
||||
</section><section class="panel"><h2>Agent access</h2><p>Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.</p>{#if canManage}<div class="actions"><Button disabled={busy || (cleaning && !host.decommission?.retryable)} onclick={() => action('/enrollment', 'POST')}>Generate installer</Button><Button disabled={busy || cleaning || !host.credentialActive || !host.connected} onclick={() => action('/credentials', 'POST', undefined, 'Credential rotation queued.')}>Rotate credential</Button><Button disabled={busy || cleaning || !host.credentialActive} onclick={() => action('/credentials', 'DELETE', undefined, 'Host credentials revoked. Generate a new installer to enroll again.')}>Revoke credential</Button></div>
|
||||
{#if enrollment}<p>Expires {new Date(enrollment.expiresAt).toLocaleString()}</p><pre>{enrollment.command}</pre>{/if}{:else}<p>Administrators manage enrollment and agent credentials.</p>{/if}
|
||||
</section></div>
|
||||
<section class="panel"><h2>Operation history</h2>{#if !operations.length}<p>No remote operations yet.</p>{:else}<div class="table-wrap"><table><thead><tr><th>Operation</th><th>State</th><th>Updated</th><th>Result</th></tr></thead><tbody>{#each operations as operation}<tr><td>{operation.kind}<small>{operation.id}</small></td><td>{operation.state}</td><td>{new Date(operation.updatedAt).toLocaleString()}</td><td>{operation.error ?? '—'}</td></tr>{/each}</tbody></table></div>{/if}</section>
|
||||
{/if}
|
||||
</section>
|
||||
<DeleteHostDialog bind:open={deleteOpen} {host} ondeleted={async (result) => { if (result.deleted) await goto('/hosts'); else { enrollment = null; await load(); } }} />
|
||||
|
||||
<style>
|
||||
.page { display:grid; gap:24px; } .panel { border:1px solid var(--border-strong); border-radius:var(--radius-md); background:var(--surface); padding:24px; min-width:0; } h2 { margin:0 0 16px; font-size:var(--text-md); text-transform:capitalize; } p, li, dd, dt, span { font-size:var(--text-sm); line-height:1.65; } p,li,dt,span { color:var(--text-secondary); } .grid { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:24px; } .row,.actions,.fields { display:flex; gap:12px; flex-wrap:wrap; } .row { justify-content:space-between; } .actions { margin-top:20px; } dl { display:grid; grid-template-columns:1fr 1.3fr; gap:12px; margin:0; } dd { margin:0; overflow-wrap:anywhere; } fieldset { display:grid; gap:16px; border:0; padding:0; margin:0; } label { display:grid; gap:8px; font-size:var(--text-sm); } input { width:100%; min-width:0; padding:10px; color:var(--text-primary); background:var(--surface-sunken); border:1px solid var(--border-control); border-radius:var(--radius-sm); } .fields>* { flex:1; min-width:120px; } pre { white-space:pre-wrap; overflow-wrap:anywhere; padding:16px; background:var(--surface-sunken); font-size:var(--text-xs); } .table-wrap { overflow:auto; } table { width:100%; border-collapse:collapse; text-align:left; font-size:var(--text-sm); } td,th { padding:12px; border-bottom:1px solid var(--border-subtle); } small { display:block; color:var(--text-secondary); font-size:var(--text-xs); margin-top:4px; } @media(max-width:900px) { .grid { grid-template-columns:1fr; } }
|
||||
|
||||
+338
-4
@@ -29,10 +29,17 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
VERSION = "1.0.0"
|
||||
VERSION = "1.1.0"
|
||||
LABEL = "io.electric-cherry"
|
||||
MAX_FILE = 32 * 1024 * 1024
|
||||
ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
|
||||
AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
|
||||
AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
|
||||
AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service")
|
||||
UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
|
||||
UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service")
|
||||
UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
|
||||
ROOT_MARKER = ".electric-cherry.json"
|
||||
|
||||
|
||||
class AgentError(Exception):
|
||||
@@ -47,6 +54,97 @@ class Rejected(AgentError):
|
||||
"""A definitive precondition failure, before any destructive action."""
|
||||
|
||||
|
||||
def systemctl(*args):
|
||||
try:
|
||||
result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
raise AgentError("Host service management is unavailable") from None
|
||||
if result.returncode:
|
||||
raise AgentError("Host service management did not complete")
|
||||
|
||||
|
||||
def protected_path(path, missing=False):
|
||||
"""Deletion authority must never come from a runtime-writable path."""
|
||||
path = Path(path)
|
||||
for current in (*reversed(path.parents), path):
|
||||
try:
|
||||
info = current.lstat()
|
||||
except FileNotFoundError:
|
||||
if missing:
|
||||
continue
|
||||
raise AgentError("Host uninstall files are unavailable; reinstall the agent") from None
|
||||
if stat_module.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
|
||||
raise AgentError("Host uninstall paths must be owned and writable only by root")
|
||||
return path
|
||||
|
||||
|
||||
def protected_file(path, missing=False):
|
||||
path = protected_path(path, missing=missing)
|
||||
try:
|
||||
if not stat_module.S_ISREG(path.lstat().st_mode):
|
||||
raise AgentError("Host uninstall metadata must be a regular file")
|
||||
except FileNotFoundError:
|
||||
if not missing:
|
||||
raise
|
||||
return path
|
||||
|
||||
|
||||
def sync_parent(path):
|
||||
descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def uninstall_root(value, identity=None):
|
||||
"""Accept only a canonical, dedicated directory with no nested mounts."""
|
||||
if not isinstance(value, str) or not re.fullmatch(r"/[A-Za-z0-9/_.-]+", value):
|
||||
raise AgentError("Host data directory is unsafe to remove")
|
||||
root = Path(value)
|
||||
forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"}
|
||||
if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden:
|
||||
raise AgentError("Host data directory is unsafe to remove")
|
||||
for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT):
|
||||
if installed == root or root in installed.parents:
|
||||
raise AgentError("Host data directory overlaps the agent installation")
|
||||
protected_path(root, missing=True)
|
||||
if root.exists():
|
||||
info = root.stat()
|
||||
if not stat_module.S_ISDIR(info.st_mode):
|
||||
raise AgentError("Host data directory is not a directory")
|
||||
if identity and [info.st_dev, info.st_ino] != identity:
|
||||
raise AgentError("Host data directory changed since deletion was authorized")
|
||||
try:
|
||||
# ProtectSystem/ReadWritePaths give the running agent a private bind
|
||||
# mount at its data directory. Inspect the host systemd namespace,
|
||||
# otherwise every legitimate installation appears to be a mount root.
|
||||
mount_lines = Path("/proc/1/mountinfo").read_text().splitlines()
|
||||
except FileNotFoundError:
|
||||
mount_lines = [] # Non-Linux tests; the installed service requires Linux.
|
||||
for line in mount_lines:
|
||||
fields = line.split()
|
||||
if len(fields) < 5:
|
||||
continue
|
||||
mount = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), fields[4])
|
||||
if mount == value or mount.startswith(value + "/"):
|
||||
raise AgentError("Unmount filesystems inside the host data directory before deletion")
|
||||
return root
|
||||
|
||||
|
||||
def uninstall_ownership(config):
|
||||
root = uninstall_root(config.get("root"))
|
||||
marker = protected_file(root / ROOT_MARKER)
|
||||
try:
|
||||
saved = json.loads(marker.read_text())
|
||||
except (OSError, ValueError):
|
||||
raise AgentError("Host data ownership is unavailable; reinstall the agent") from None
|
||||
expected = {"version": 1, "root": str(root), "hostId": config.get("hostId"), "controller": config.get("controller", "").rstrip("/")}
|
||||
if saved != expected or not expected["hostId"]:
|
||||
raise AgentError("Host data ownership does not match this enrollment")
|
||||
return root
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
@@ -275,7 +373,7 @@ class Docker:
|
||||
raise AgentError("Docker command unavailable or timed out") from None
|
||||
if result.returncode:
|
||||
# Never include arbitrary Docker stderr: create errors may echo secrets.
|
||||
if tolerate_missing and (b"No such object" in (result.stderr or b"") or b"No such container" in (result.stderr or b"")):
|
||||
if tolerate_missing and any(message in (result.stderr or b"") for message in (b"No such object", b"No such container", b"No such image")):
|
||||
return None
|
||||
raise AgentError("Docker rejected the requested operation")
|
||||
return result.stdout.decode("utf-8", "replace")
|
||||
@@ -285,6 +383,188 @@ class Docker:
|
||||
return json.loads(raw)[0] if raw is not None else None
|
||||
|
||||
|
||||
class Uninstaller:
|
||||
"""A separate systemd service owns cleanup after stopping the host agent.
|
||||
|
||||
Its root-only request and executable stay on disk until the controller
|
||||
acknowledges data removal. Completion is reported only after removing those
|
||||
final files. A lost final response can be retried from memory; a power loss
|
||||
in that last window deliberately leaves controller status unconfirmed.
|
||||
"""
|
||||
|
||||
def __init__(self, request_path, docker=None):
|
||||
if Path(request_path) != UNINSTALL_REQUEST:
|
||||
raise AgentError("Invalid host uninstall request location")
|
||||
protected_file(UNINSTALL_REQUEST)
|
||||
self.job = json.loads(UNINSTALL_REQUEST.read_text())
|
||||
self.docker = docker or Docker()
|
||||
self.root = uninstall_root(self.job.get("root"), self.job.get("rootIdentity"))
|
||||
host_id = identifier(self.job.get("hostId"))
|
||||
operation_id = identifier(self.job.get("operationId"))
|
||||
parsed = urllib.parse.urlsplit(self.job.get("controller", ""))
|
||||
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise AgentError("Host uninstall controller must use HTTPS")
|
||||
if self.job.get("receiptUrl") != "/api/agent/decommission/" + operation_id or not isinstance(self.job.get("receiptToken"), str) or not self.job["receiptToken"]:
|
||||
raise AgentError("Invalid host uninstall receipt")
|
||||
if not isinstance(self.job.get("rootIdentity"), list) or len(self.job["rootIdentity"]) != 2 or not all(isinstance(value, int) and not isinstance(value, bool) for value in self.job["rootIdentity"]):
|
||||
raise AgentError("Invalid host data ownership proof")
|
||||
if self.root.exists():
|
||||
# The authorized device/inode is persisted before deletion so a
|
||||
# crash halfway through rmtree does not require a deleted marker.
|
||||
if not self.job.get("removalStarted"):
|
||||
uninstall_ownership({"root": str(self.root), "controller": self.job["controller"], "hostId": host_id})
|
||||
|
||||
def save(self, **changes):
|
||||
self.job.update(changes)
|
||||
atomic_json(UNINSTALL_REQUEST, self.job)
|
||||
|
||||
def receipt(self, stage, error=None):
|
||||
payload = {"stage": stage}
|
||||
if error:
|
||||
payload["error"] = error
|
||||
request = urllib.request.Request(self.job["controller"].rstrip("/") + self.job["receiptUrl"], data=json.dumps(payload).encode(), headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.job["receiptToken"], "User-Agent": "electric-cherry-agent/" + VERSION}, method="POST")
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, hdrs, newurl):
|
||||
raise AgentError("Controller redirects are not allowed")
|
||||
try:
|
||||
with urllib.request.build_opener(NoRedirect()).open(request, timeout=35) as response:
|
||||
if json.load(response).get("acknowledged") is not True:
|
||||
raise AgentError("Controller has not acknowledged host removal")
|
||||
except (OSError, ValueError):
|
||||
raise AgentError("Host removal receipt is waiting for controller contact") from None
|
||||
|
||||
def containers(self):
|
||||
ids = self.docker.call("container", "ls", "--all", "--quiet", "--no-trunc").splitlines()
|
||||
values = []
|
||||
for container_id in ids:
|
||||
info = self.docker.inspect(container_id)
|
||||
if info:
|
||||
values.append(info)
|
||||
return values
|
||||
|
||||
def owned_container(self, info):
|
||||
labels = info.get("Config", {}).get("Labels", {}) or {}
|
||||
if labels.get(LABEL + ".managed") != "true":
|
||||
return False
|
||||
if labels.get(LABEL + ".host") not in (None, self.job["hostId"]):
|
||||
return False
|
||||
try:
|
||||
instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", "")))
|
||||
except (AgentError, ValueError, TypeError):
|
||||
return False
|
||||
expected = str(self.root / "instances" / instance_id / str(generation) / "app")
|
||||
# Legacy agents did not label the host; its exact managed bind mount
|
||||
# proves ownership without touching another installation's containers.
|
||||
return any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", []))
|
||||
|
||||
def uses_root(self, info):
|
||||
for mount in info.get("Mounts", []):
|
||||
if mount.get("Type") != "bind":
|
||||
continue
|
||||
source = Path(mount.get("Source", "/")).resolve()
|
||||
if source == self.root or self.root in source.parents:
|
||||
return True
|
||||
return False
|
||||
|
||||
def remove_docker_resources(self):
|
||||
containers = self.containers()
|
||||
if any(self.uses_root(info) and not self.owned_container(info) for info in containers):
|
||||
raise AgentError("An unrelated container uses the host data directory; detach it before retrying deletion")
|
||||
owned = [info for info in containers if self.owned_container(info)]
|
||||
if any(info.get("State", {}).get("Running") is not False or info.get("State", {}).get("Status") not in ("created", "exited", "dead") for info in owned):
|
||||
raise AgentError("A managed game container is still running or its state is unknown; retire it before deleting this host")
|
||||
for info in owned:
|
||||
# Do not force removal: Docker must also reject a container that
|
||||
# starts again after inspection rather than interrupting players.
|
||||
self.docker.call("container", "rm", "--volumes", info["Id"], timeout=120, tolerate_missing=True)
|
||||
remaining = self.containers()
|
||||
if any(self.uses_root(info) for info in remaining):
|
||||
raise AgentError("Containers still use the host data directory")
|
||||
# Never prune Docker globally. Only remove this agent's recorded image
|
||||
# references, and preserve images used by any unrelated container.
|
||||
used_images = {info.get("Image") for info in remaining}
|
||||
images = self.job.get("runtimeImages", {})
|
||||
if not isinstance(images, dict):
|
||||
raise AgentError("Managed runtime image ownership is unavailable")
|
||||
for reference, expected_id in images.items():
|
||||
if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,511}", reference) or not isinstance(expected_id, str):
|
||||
raise AgentError("Managed runtime image ownership is invalid")
|
||||
raw = self.docker.call("image", "inspect", reference, tolerate_missing=True)
|
||||
if raw is None:
|
||||
continue
|
||||
found = json.loads(raw)
|
||||
if not found or found[0].get("Id") != expected_id or expected_id in used_images:
|
||||
continue
|
||||
self.docker.call("image", "rm", reference, tolerate_missing=True)
|
||||
|
||||
def remove_data(self):
|
||||
protected_path(AGENT_CONFIG.parent, missing=True)
|
||||
if not self.job.get("agentStopped"):
|
||||
systemctl("disable", "--now", AGENT_UNIT.name)
|
||||
self.save(agentStopped=True)
|
||||
self.remove_docker_resources()
|
||||
root = uninstall_root(str(self.root), self.job["rootIdentity"])
|
||||
# This checkpoint also authorizes safe continuation if a power loss
|
||||
# removes the ownership marker before the rest of the tree.
|
||||
self.save(removalStarted=True)
|
||||
if root.exists():
|
||||
if not shutil.rmtree.avoids_symlink_attacks:
|
||||
raise AgentError("This platform cannot safely remove managed host files")
|
||||
shutil.rmtree(root)
|
||||
sync_parent(root)
|
||||
for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT):
|
||||
protected_file(path, missing=True)
|
||||
path.unlink(missing_ok=True)
|
||||
sync_parent(path)
|
||||
systemctl("daemon-reload")
|
||||
self.save(dataRemoved=True)
|
||||
|
||||
def remove_helper(self):
|
||||
# Disable does not stop this process: it must outlive its own unit and
|
||||
# executable to deliver the final receipt after the files are gone.
|
||||
protected_file(UNINSTALL_UNIT, missing=True)
|
||||
if UNINSTALL_UNIT.exists():
|
||||
systemctl("disable", UNINSTALL_UNIT.name)
|
||||
UNINSTALL_UNIT.unlink()
|
||||
sync_parent(UNINSTALL_UNIT)
|
||||
systemctl("daemon-reload")
|
||||
protected_file(UNINSTALL_EXECUTABLE, missing=True)
|
||||
UNINSTALL_EXECUTABLE.unlink(missing_ok=True)
|
||||
sync_parent(UNINSTALL_EXECUTABLE)
|
||||
protected_path(UNINSTALL_REQUEST.parent, missing=True)
|
||||
if UNINSTALL_REQUEST.parent.exists():
|
||||
shutil.rmtree(UNINSTALL_REQUEST.parent)
|
||||
sync_parent(UNINSTALL_REQUEST.parent)
|
||||
|
||||
def run(self):
|
||||
if not self.job.get("dataRemoved"):
|
||||
try:
|
||||
self.remove_data()
|
||||
except (AgentError, OSError, ValueError, KeyError) as error:
|
||||
with contextlib.suppress(AgentError):
|
||||
self.receipt("failed", str(error) if isinstance(error, AgentError) else "Host cleanup could not finish. Check the uninstall service and resolve filesystem errors; cleanup will retry.")
|
||||
raise
|
||||
self.receipt("cleaned")
|
||||
while True:
|
||||
try:
|
||||
self.remove_helper()
|
||||
break
|
||||
except (AgentError, OSError, ValueError):
|
||||
# Once its unit/executable has been unlinked systemd can no
|
||||
# longer restart this worker. Finish idempotently in memory.
|
||||
with contextlib.suppress(AgentError):
|
||||
self.receipt("failed", "Host files were removed, but removing the uninstall service is still retrying")
|
||||
time.sleep(15)
|
||||
# The controller never marks a host deleted on the earlier receipt.
|
||||
# No credentials or retry state are left on disk at this point.
|
||||
while True:
|
||||
try:
|
||||
self.receipt("completed")
|
||||
return
|
||||
except AgentError:
|
||||
time.sleep(15)
|
||||
|
||||
|
||||
class Agent:
|
||||
def __init__(self, config, docker=None):
|
||||
self.config = config
|
||||
@@ -300,6 +580,43 @@ class Agent:
|
||||
self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
|
||||
self.executing_id = None
|
||||
|
||||
def can_uninstall(self):
|
||||
try:
|
||||
if Path(self.config_path) != AGENT_CONFIG:
|
||||
return False
|
||||
uninstall_ownership(self.config)
|
||||
protected_file(UNINSTALL_EXECUTABLE)
|
||||
protected_file(UNINSTALL_UNIT)
|
||||
return True
|
||||
except (AgentError, OSError, ValueError):
|
||||
return False
|
||||
|
||||
def decommission(self, operation):
|
||||
payload = operation.get("payload", {})
|
||||
if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]:
|
||||
raise Rejected("Host removal identity does not match this agent")
|
||||
if payload.get("receiptUrl") != "/api/agent/decommission/" + identifier(operation["id"]) or not isinstance(payload.get("receiptToken"), str) or not payload["receiptToken"]:
|
||||
raise Rejected("Invalid host removal receipt")
|
||||
if not self.can_uninstall():
|
||||
raise Rejected("Reinstall the agent to enable safe removal of its dedicated host files")
|
||||
root = uninstall_ownership(self.config)
|
||||
info = root.stat()
|
||||
protected_file(UNINSTALL_REQUEST, missing=True)
|
||||
aliases_path = root / "agent" / "runtime-images.json"
|
||||
protected_file(aliases_path, missing=True)
|
||||
aliases = json.loads(aliases_path.read_text()) if aliases_path.exists() else {}
|
||||
job = {"hostId": payload["hostId"], "operationId": operation["id"], "receiptUrl": payload["receiptUrl"], "receiptToken": payload["receiptToken"], "controller": self.config["controller"].rstrip("/"), "root": str(root), "rootIdentity": [info.st_dev, info.st_ino], "runtimeImages": aliases}
|
||||
if UNINSTALL_REQUEST.exists():
|
||||
previous = json.loads(UNINSTALL_REQUEST.read_text())
|
||||
if any(previous.get(key) != job[key] for key in ("hostId", "operationId", "receiptUrl", "receiptToken", "controller", "root", "rootIdentity")):
|
||||
raise Rejected("A different host removal is already pending")
|
||||
self.checkpoint({"phase": "uninstall-service"})
|
||||
if not UNINSTALL_REQUEST.exists():
|
||||
atomic_json(UNINSTALL_REQUEST, job)
|
||||
systemctl("enable", "--now", "--no-block", UNINSTALL_UNIT.name)
|
||||
self.stopping.set()
|
||||
return {"decommissioning": True, "message": "The uninstall service will report completion after host files and the agent are removed"}
|
||||
|
||||
def instance_dir(self, instance_id, generation):
|
||||
instance_id, generation = instance_identity(instance_id, generation)
|
||||
return self.root / "instances" / instance_id / str(generation)
|
||||
@@ -416,7 +733,7 @@ class Agent:
|
||||
for line in Path("/etc/os-release").read_text().splitlines():
|
||||
if line.startswith("PRETTY_NAME="):
|
||||
os_name = line.split("=", 1)[1].strip('"')
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
|
||||
|
||||
def safe_to_disrupt(self, instance_id, generation):
|
||||
health = self.inspect(instance_id, generation)
|
||||
@@ -525,6 +842,8 @@ class Agent:
|
||||
name = self.container_name(instance_id, generation)
|
||||
spec_sha = digest(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode())
|
||||
args = ["container", "create", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)]
|
||||
if self.config.get("hostId"):
|
||||
args.extend(["--label", LABEL + ".host=" + identifier(self.config["hostId"])])
|
||||
bind_arg = "[" + bind + "]" if ":" in bind else bind
|
||||
for protocol in ("udp", "tcp"):
|
||||
args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
|
||||
@@ -923,6 +1242,8 @@ class Agent:
|
||||
kind, payload = operation.get("kind"), operation.get("payload", {})
|
||||
if not isinstance(kind, str) or not isinstance(payload, dict):
|
||||
raise AgentError("Invalid operation")
|
||||
if kind == "host.decommission":
|
||||
return self.decommission(operation)
|
||||
if kind == "credentials.rotate":
|
||||
pending = "ec_host_" + base64.urlsafe_b64encode(os.urandom(32)).rstrip(b"=").decode()
|
||||
self.config["pendingCredential"] = pending
|
||||
@@ -997,7 +1318,7 @@ class Agent:
|
||||
return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message)
|
||||
finally:
|
||||
self.executing_id = None
|
||||
return self.journal.update(operation["id"], "completed", result)
|
||||
return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result)
|
||||
|
||||
def recover_credential(self):
|
||||
pending = self.config.get("pendingCredential")
|
||||
@@ -1024,6 +1345,13 @@ class Agent:
|
||||
instance_id, generation = operation.get("instanceId"), operation.get("generation")
|
||||
checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
|
||||
try:
|
||||
if kind == "host.decommission":
|
||||
self.executing_id = operation["id"]
|
||||
try:
|
||||
result = self.decommission(operation)
|
||||
return self.journal.update(operation["id"], "running", result)
|
||||
finally:
|
||||
self.executing_id = None
|
||||
if kind == "credentials.rotate":
|
||||
if not self.config.get("pendingCredential") or self.recover_credential():
|
||||
return self.journal.update(operation["id"], "completed", {"rotated": True, "reconciled": True})
|
||||
@@ -1183,8 +1511,14 @@ def main():
|
||||
parser.add_argument("--token-file")
|
||||
parser.add_argument("--root", default="/var/lib/electric-cherry")
|
||||
parser.add_argument("--force-enroll", action="store_true")
|
||||
parser.add_argument("--uninstall", metavar="REQUEST", help="Run the installed host removal service")
|
||||
parser.add_argument("--version", action="version", version=VERSION)
|
||||
args = parser.parse_args()
|
||||
if args.uninstall:
|
||||
if os.geteuid() != 0 or args.enroll:
|
||||
raise AgentError("Host uninstall must run as the dedicated root service")
|
||||
Uninstaller(args.uninstall).run()
|
||||
return
|
||||
if args.enroll:
|
||||
if not args.controller or not args.token_file:
|
||||
parser.error("Enrollment needs --controller and --token-file")
|
||||
|
||||
+120
-7
@@ -21,7 +21,7 @@ done
|
||||
case "$controller" in https://*) ;; *) echo '--controller must be an HTTPS URL.' >&2; exit 2 ;; esac
|
||||
[ -n "$enrollment_token" ] || { echo '--token is required.' >&2; exit 2; }
|
||||
case "$data_dir" in /*) ;; *) echo '--data-dir must be absolute.' >&2; exit 2 ;; esac
|
||||
case "$data_dir" in *[!a-zA-Z0-9/_.-]*|/|/etc|/usr|/var|/var/lib) echo 'Choose a dedicated data directory.' >&2; exit 2 ;; esac
|
||||
case "$data_dir" in *[!a-zA-Z0-9/_.-]*|*/../*|*/./*|*/..|*/.|*//*|*/) echo 'Choose a canonical, dedicated data directory.' >&2; exit 2 ;; esac
|
||||
[ ! -L "$data_dir" ] || { echo 'Data directory must not be a symlink.' >&2; exit 2; }
|
||||
. /etc/os-release
|
||||
[ "${ID:-}" = debian ] || { echo 'Supported platform: Debian 12 or 13.' >&2; exit 1; }
|
||||
@@ -32,13 +32,82 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; }
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends ca-certificates curl python3
|
||||
python3 - "$controller" "$data_dir" <<'PY'
|
||||
import json, pathlib, sys
|
||||
config = pathlib.Path('/etc/electric-cherry/agent.json')
|
||||
if config.exists():
|
||||
saved = json.loads(config.read_text())
|
||||
if saved.get('controller', '').rstrip('/') != sys.argv[1].rstrip('/') or pathlib.Path(saved.get('root', '')).resolve() != pathlib.Path(sys.argv[2]).resolve():
|
||||
python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY'
|
||||
# INSTALLATION_PREFLIGHT: this runs before touching managed directories or services.
|
||||
import json, pathlib, re, stat, sys
|
||||
|
||||
controller, raw_root, config_name = sys.argv[1:]
|
||||
root = pathlib.Path(raw_root)
|
||||
config = pathlib.Path(config_name)
|
||||
reserved = {
|
||||
'/', '/bin', '/boot', '/dev', '/etc', '/home', '/lib', '/lib64',
|
||||
'/media', '/mnt', '/opt', '/proc', '/root', '/run', '/sbin', '/srv',
|
||||
'/sys', '/tmp', '/usr', '/usr/local', '/usr/local/bin', '/usr/local/lib',
|
||||
'/usr/local/libexec', '/var', '/var/cache', '/var/lib', '/var/log', '/var/tmp',
|
||||
}
|
||||
if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root)
|
||||
or str(root) != raw_root or any(part in ('.', '..') for part in raw_root.split('/')[1:])
|
||||
or len(root.parts) < 3 or raw_root in reserved or config.parent == root or config.parent in root.parents):
|
||||
raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.')
|
||||
installed_paths = [config,
|
||||
pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'),
|
||||
pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'),
|
||||
pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'),
|
||||
pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service')]
|
||||
if any(root == path or root in path.parents for path in installed_paths):
|
||||
raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.')
|
||||
|
||||
def trusted_directories(path):
|
||||
for current in [*reversed(path.parents), path]:
|
||||
try:
|
||||
info = current.lstat()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
|
||||
raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.')
|
||||
|
||||
def trusted_json(path):
|
||||
info = path.lstat()
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
|
||||
raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.')
|
||||
value = json.loads(path.read_text())
|
||||
if not isinstance(value, dict):
|
||||
raise SystemExit('Existing installation metadata is invalid.')
|
||||
return value
|
||||
|
||||
trusted_directories(root)
|
||||
trusted_directories(config.parent)
|
||||
mountinfo = pathlib.Path('/proc/1/mountinfo')
|
||||
if mountinfo.exists():
|
||||
for line in mountinfo.read_text().splitlines():
|
||||
fields = line.split()
|
||||
if len(fields) < 5:
|
||||
continue
|
||||
mount = re.sub(r'\\([0-7]{3})', lambda match: chr(int(match[1], 8)), fields[4])
|
||||
if mount == raw_root or mount.startswith(raw_root + '/'):
|
||||
raise SystemExit('Use a dedicated subdirectory of the mounted disk, with no filesystems mounted inside it, so host deletion can remove the entire data directory.')
|
||||
request = config.with_name('decommission.json')
|
||||
if request.exists() or request.is_symlink():
|
||||
raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.')
|
||||
saved = None
|
||||
if config.exists() or config.is_symlink():
|
||||
saved = trusted_json(config)
|
||||
if saved.get('controller', '').rstrip('/') != controller.rstrip('/') or saved.get('root') != raw_root:
|
||||
raise SystemExit('Existing enrollment uses a different controller or data directory. Preserve the existing installation and migrate it explicitly.')
|
||||
if not saved.get('hostId') or not saved.get('credential'):
|
||||
raise SystemExit('Existing enrollment is incomplete. Repair its credentials before upgrading.')
|
||||
marker = root / '.electric-cherry.json'
|
||||
if marker.exists() or marker.is_symlink():
|
||||
owned = trusted_json(marker)
|
||||
if not saved or owned != {'version': 1, 'root': raw_root, 'hostId': saved['hostId'], 'controller': controller.rstrip('/')}:
|
||||
raise SystemExit('Data directory ownership does not match this enrollment. Preserve it and migrate explicitly.')
|
||||
elif root.exists() and any(root.iterdir()):
|
||||
# Upgrade installations predating ownership markers only when the entire
|
||||
# root has the known layout. Never adopt a directory of unrelated files.
|
||||
if not saved or any(path.name not in {'agent', 'instances', 'libraries'} for path in root.iterdir()):
|
||||
raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.')
|
||||
for directory in ('agent', 'instances', 'libraries'):
|
||||
trusted_directories(root / directory)
|
||||
PY
|
||||
if ! command -v docker >/dev/null; then
|
||||
if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then
|
||||
@@ -76,9 +145,36 @@ install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-
|
||||
install -d -m 0755 /usr/local/libexec
|
||||
install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py
|
||||
}
|
||||
# A separate worker survives stopping/removing the agent service. Its only
|
||||
# entry point is a fixed, authenticated deletion request prepared by the agent.
|
||||
install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py
|
||||
printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token"
|
||||
unset enrollment_token
|
||||
python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"}
|
||||
python3 - "$data_dir" /etc/electric-cherry/agent.json <<'PY'
|
||||
# INSTALLATION_OWNERSHIP: bind deletion authority to this exact enrollment/root.
|
||||
import json, os, pathlib, sys, uuid
|
||||
root = pathlib.Path(sys.argv[1])
|
||||
config = json.loads(pathlib.Path(sys.argv[2]).read_text())
|
||||
if config.get('root') != str(root) or not config.get('hostId') or not config.get('controller'):
|
||||
raise SystemExit('Enrollment did not establish ownership of this data directory.')
|
||||
marker = root / '.electric-cherry.json'
|
||||
temporary = root / ('.electric-cherry.' + uuid.uuid4().hex + '.tmp')
|
||||
try:
|
||||
with temporary.open('x') as stream:
|
||||
os.chmod(temporary, 0o600)
|
||||
json.dump({'version': 1, 'root': str(root), 'hostId': config['hostId'], 'controller': config['controller'].rstrip('/')}, stream)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, marker)
|
||||
directory_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
PY
|
||||
cat > /etc/systemd/system/electric-cherry-agent.service <<EOF
|
||||
[Unit]
|
||||
Description=Electric Cherry Host Agent
|
||||
@@ -99,6 +195,23 @@ ProtectHome=true
|
||||
PrivateTmp=true
|
||||
ReadWritePaths=$data_dir /etc/electric-cherry
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
cat > /etc/systemd/system/electric-cherry-uninstall.service <<'EOF'
|
||||
[Unit]
|
||||
Description=Remove an Electric Cherry Host Installation
|
||||
After=network-online.target docker.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-uninstall.py --uninstall /etc/electric-cherry/decommission.json
|
||||
Restart=on-failure
|
||||
RestartSec=15
|
||||
TimeoutStartSec=0
|
||||
UMask=0077
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"version": "1.0.0",
|
||||
"version": "1.1.0",
|
||||
"protocolVersion": 1,
|
||||
"platform": "debian-amd64",
|
||||
"sha256": "2b6e6c2cc4c949e8f980c147aa6b8725844b7c320d7627e0328570d23c985a43"
|
||||
"sha256": "67046a3d81f251e148f2323cade255ff584a622750a8c05ee27f96c70e09fcc7"
|
||||
}
|
||||
|
||||
@@ -46,10 +46,11 @@ describe('management route permissions', () => {
|
||||
});
|
||||
|
||||
it.each([
|
||||
['/api/hosts', 'POST'], ['/api/hosts/[id]', 'PATCH'], ['/api/hosts/[id]/enrollment', 'POST'],
|
||||
['/api/hosts', 'POST'], ['/api/hosts/[id]', 'PATCH'], ['/api/hosts/[id]', 'DELETE'], ['/api/hosts/[id]/enrollment', 'POST'],
|
||||
['/api/hosts/[id]/credentials', 'DELETE'], ['/api/hosts/[id]/credentials', 'POST'],
|
||||
['/api/hosts/[id]/operations', 'POST'], ['/api/settings', 'PUT'], ['/api/game-files', 'POST'],
|
||||
['/api/game-files/[id]/download', 'POST'], ['/api/maintenance', 'POST'], ['/api/watchdog', 'POST'], ['/api/plutonium', 'POST']
|
||||
['/api/game-files/[id]/download', 'POST'], ['/api/game-files/[id]/upload/files/[...path]', 'PUT'],
|
||||
['/api/game-files/[id]/upload/complete', 'POST'], ['/api/maintenance', 'POST'], ['/api/watchdog', 'POST'], ['/api/plutonium', 'POST']
|
||||
])('protects host, game-file, and global policy mutations at %s', (route, method) => {
|
||||
expect(routePermission(route, method)).toBe('admin');
|
||||
expect(allows('operator', route, method)).toBe(false);
|
||||
@@ -91,6 +92,7 @@ describe('authentication and machine boundaries', () => {
|
||||
it.each([
|
||||
['/api/agent/enroll', 'POST'], ['/api/agent/heartbeat', 'POST'], ['/api/agent/operations', 'GET'],
|
||||
['/api/agent/operations/[id]', 'POST'], ['/api/agent/libraries/[id]/files/[...path]', 'GET'],
|
||||
['/api/agent/decommission/[id]', 'POST'],
|
||||
['/api/agent/imports/[id]/files/[...path]', 'PUT'], ['/api/agent/imports/[id]/complete', 'POST'],
|
||||
['/api/agent/install.sh', 'GET'], ['/api/agent/release/[file]', 'GET'], ['/api/telemetry', 'POST']
|
||||
])('preserves endpoint-specific machine authentication for %s', (route, method) => {
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdir, mkdtemp, readFile, rename, rm, stat } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const state = vi.hoisted(() => ({ handle: null as Database.Database | null }));
|
||||
vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle }));
|
||||
import {
|
||||
browserUploadStatus, createLibraryImport, createLibraryUpload, getHostLibraryCache,
|
||||
getLibraryVersion, listLibraries, publishBrowserUpload, queueLibraryDownload,
|
||||
requestBrowserPublication, requireReadyLibrary, updateHostLibraryCaches, uploadBrowserFile
|
||||
} from '../../src/lib/server/game-files/index.js';
|
||||
import { createEnrollment, createHost, enrollHost, listHosts } from '../../src/lib/server/hosts/index.js';
|
||||
import { manifestChecksum } from '../../src/lib/server/game-files/storage.js';
|
||||
import { serveGameFile } from '../../src/lib/server/game-files/http.js';
|
||||
import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js';
|
||||
|
||||
let directory: string;
|
||||
const modified = 1_700_000_000_000;
|
||||
const bytes = Buffer.from('verified Plutonium game content');
|
||||
const sha = (value: Uint8Array) => createHash('sha256').update(value).digest('hex');
|
||||
const file = (path = 'zone/base.ff', content = bytes) => ({ path, size: content.length, lastModified: modified });
|
||||
const upload = (id: string, path: string, content: Buffer, offset = 0, checksum = sha(content)) =>
|
||||
uploadBrowserFile(id, path, { offset, size: content.length, sha256: checksum }, new Response(new Uint8Array(content)).body);
|
||||
const begin = (game: 't5' | 't6' = 't6', files = [file()], version = '1') =>
|
||||
createLibraryUpload({ game, name: `Plutonium ${game.toUpperCase()}`, version, description: 'Uploaded game folder', files });
|
||||
|
||||
beforeEach(async () => {
|
||||
directory = await mkdtemp(join(tmpdir(), 'ec-browser-library-'));
|
||||
state.handle = new Database(join(directory, 'controller.sqlite'));
|
||||
state.handle.pragma('foreign_keys=ON');
|
||||
vi.stubEnv('GAME_FILES_ROOT', join(directory, 'libraries'));
|
||||
});
|
||||
afterEach(async () => {
|
||||
state.handle?.close(); state.handle = null;
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
function host() {
|
||||
const created = createHost({ name: 'Fresh download target' });
|
||||
const { credential } = enrollHost(createEnrollment(created.id).token, {
|
||||
protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3
|
||||
});
|
||||
return { id: created.id, credential };
|
||||
}
|
||||
|
||||
describe('browser folder uploads', () => {
|
||||
it.each(['t5', 't6'] as const)('publishes %s from a browser without an enrolled host or DLC declarations', async game => {
|
||||
expect(listHosts()).toEqual([]);
|
||||
const initial = begin(game);
|
||||
expect(initial).toMatchObject({ state: 'importing', operationId: null, sourceHostId: null, sourceVolume: null });
|
||||
expect(initial).not.toHaveProperty('dlc');
|
||||
expect(await browserUploadStatus(initial.id)).toMatchObject({
|
||||
status: 'pending', files: [{ ...file(), receivedBytes: 0, complete: false }]
|
||||
});
|
||||
await upload(initial.id, file().path, bytes);
|
||||
const accepted = await requestBrowserPublication(initial.id);
|
||||
expect(['verifying', 'published']).toContain(accepted.status);
|
||||
const ready = await publishBrowserUpload(initial.id);
|
||||
expect(ready).toMatchObject({ state: 'ready', sizeBytes: bytes.length, fileCount: 1 });
|
||||
expect(ready.files).toEqual([{ path: file().path, size: bytes.length, sha256: sha(bytes) }]);
|
||||
expect(ready.manifestSha256).toBe(manifestChecksum(ready.files));
|
||||
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
|
||||
expect(await browserUploadStatus(initial.id)).toMatchObject({ status: 'published' });
|
||||
expect(listLibraries()[0].versions[0]).not.toHaveProperty('dlc');
|
||||
expect(listHosts()).toEqual([]);
|
||||
});
|
||||
|
||||
it('preserves nested relative contents and derives a canonical manifest in path order', async () => {
|
||||
const selected = [file('zone/english/a.ff'), file('main/iw_00.iwd'), file('empty.txt', Buffer.alloc(0))];
|
||||
const initial = begin('t6', selected);
|
||||
for (const item of selected) await upload(initial.id, item.path, item.size ? bytes : Buffer.alloc(0));
|
||||
const ready = await publishBrowserUpload(initial.id);
|
||||
expect(ready.files.map(item => item.path)).toEqual(['empty.txt', 'main/iw_00.iwd', 'zone/english/a.ff']);
|
||||
expect(ready.files[0].sha256).toBe(sha(Buffer.alloc(0)));
|
||||
expect(ready.manifestSha256).toBe(manifestChecksum(ready.files));
|
||||
expect(await readFile(join(ready.storageLocation, 'zone/english/a.ff'))).toEqual(bytes);
|
||||
});
|
||||
|
||||
it('rejects incomplete publication and allows the remaining files to finish the same version', async () => {
|
||||
const initial = begin('t5', [file('zone/first.ff'), file('zone/second.ff')]);
|
||||
await upload(initial.id, 'zone/first.ff', bytes);
|
||||
await expect(publishBrowserUpload(initial.id)).rejects.toThrow();
|
||||
expect(getLibraryVersion(initial.id)?.state).not.toBe('ready');
|
||||
const hidden = await serveGameFile(new Request('https://ec.test/files'), initial.id, 'zone/first.ff');
|
||||
expect(hidden.status).toBe(404);
|
||||
await upload(initial.id, 'zone/second.ff', bytes);
|
||||
const ready = await publishBrowserUpload(initial.id);
|
||||
expect(ready.state).toBe('ready');
|
||||
expect(ready.id).toBe(initial.id);
|
||||
});
|
||||
|
||||
it('does not acknowledge corrupt chunks and permits a verified retry', async () => {
|
||||
const initial = begin();
|
||||
await expect(upload(initial.id, file().path, bytes, 0, '0'.repeat(64))).rejects.toThrow();
|
||||
expect((await browserUploadStatus(initial.id)).files[0]).toMatchObject({ receivedBytes: 0, complete: false });
|
||||
await expect(uploadBrowserFile(initial.id, file().path, {
|
||||
offset: 0, size: bytes.length, sha256: sha(bytes)
|
||||
}, new Response(bytes.subarray(0, -1)).body)).rejects.toThrow();
|
||||
expect((await browserUploadStatus(initial.id)).files[0].receivedBytes).toBe(0);
|
||||
await upload(initial.id, file().path, bytes);
|
||||
expect((await browserUploadStatus(initial.id)).files[0]).toMatchObject({ receivedBytes: bytes.length, complete: true });
|
||||
expect((await publishBrowserUpload(initial.id)).state).toBe('ready');
|
||||
});
|
||||
|
||||
it('surfaces asynchronous incomplete publication as retryable failure', async () => {
|
||||
const initial = begin();
|
||||
await requestBrowserPublication(initial.id);
|
||||
await vi.waitFor(async () => expect((await browserUploadStatus(initial.id)).status).toBe('failed'));
|
||||
expect((await browserUploadStatus(initial.id)).error).toBeTruthy();
|
||||
await upload(initial.id, file().path, bytes);
|
||||
expect(await browserUploadStatus(initial.id)).toMatchObject({ status: 'pending', error: null });
|
||||
await requestBrowserPublication(initial.id);
|
||||
expect((await publishBrowserUpload(initial.id)).state).toBe('ready');
|
||||
});
|
||||
|
||||
it('resumes from the verified chunk boundary and safely retries a lost acknowledgement', async () => {
|
||||
const sizeProbe = begin();
|
||||
const chunkSize = (await browserUploadStatus(sizeProbe.id)).chunkSize;
|
||||
expect(chunkSize).toBeGreaterThan(0);
|
||||
const large = Buffer.alloc(chunkSize + 17, 0x41);
|
||||
const initial = begin('t6', [file('main/archive.iwd', large)], 'large');
|
||||
const first = large.subarray(0, chunkSize);
|
||||
await upload(initial.id, 'main/archive.iwd', first);
|
||||
expect((await browserUploadStatus(initial.id)).files[0]).toMatchObject({ receivedBytes: chunkSize, complete: false });
|
||||
state.handle!.close();
|
||||
state.handle = new Database(join(directory, 'controller.sqlite'));
|
||||
state.handle.pragma('foreign_keys=ON');
|
||||
await expect(upload(initial.id, 'main/archive.iwd', first)).rejects.toMatchObject({ receivedBytes: chunkSize });
|
||||
expect((await browserUploadStatus(initial.id)).files[0].receivedBytes).toBe(chunkSize);
|
||||
await upload(initial.id, 'main/archive.iwd', large.subarray(chunkSize), chunkSize);
|
||||
const ready = await publishBrowserUpload(initial.id);
|
||||
expect(ready.files[0].sha256).toBe(sha(large));
|
||||
expect((await readFile(join(ready.storageLocation, 'main/archive.iwd'))).equals(large)).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects unknown file paths, traversal, and chunks outside the declared file', async () => {
|
||||
const initial = begin();
|
||||
await expect(upload(initial.id, 'zone/unselected.ff', bytes)).rejects.toThrow();
|
||||
await expect(upload(initial.id, '../escape.ff', bytes)).rejects.toThrow();
|
||||
await expect(upload(initial.id, file().path, bytes, bytes.length + 1)).rejects.toThrow();
|
||||
await expect(upload(initial.id, file().path, Buffer.concat([bytes, bytes]))).rejects.toThrow();
|
||||
expect((await browserUploadStatus(initial.id)).files[0].receivedBytes).toBe(0);
|
||||
});
|
||||
|
||||
it('keeps ready bytes and version metadata immutable and requires a new version label', async () => {
|
||||
const initial = begin(); await upload(initial.id, file().path, bytes);
|
||||
const ready = await publishBrowserUpload(initial.id);
|
||||
await expect(upload(initial.id, file().path, bytes)).rejects.toThrow();
|
||||
expect(() => state.handle!.prepare('UPDATE game_file_versions SET description=? WHERE id=?').run('changed', ready.id)).toThrow(/immutable/);
|
||||
expect(() => begin()).toThrow(/already exists/);
|
||||
expect((await publishBrowserUpload(initial.id)).manifestSha256).toBe(ready.manifestSha256);
|
||||
expect(begin('t6', [file()], '2').id).not.toBe(ready.id);
|
||||
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
|
||||
});
|
||||
|
||||
it('recovers publication after the verified directory was renamed but the ready commit was interrupted', async () => {
|
||||
const initial = begin(); await upload(initial.id, file().path, bytes);
|
||||
const files = [{ path: file().path, size: bytes.length, sha256: sha(bytes) }];
|
||||
const checksum = manifestChecksum(files);
|
||||
state.handle!.transaction(() => {
|
||||
state.handle!.prepare('UPDATE game_file_versions SET manifest=?,manifest_sha256=? WHERE id=?')
|
||||
.run(JSON.stringify(files), checksum, initial.id);
|
||||
state.handle!.prepare('INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)')
|
||||
.run(initial.id, file().path, bytes.length, sha(bytes));
|
||||
})();
|
||||
await mkdir(dirname(initial.storageLocation), { recursive: true });
|
||||
await rename(join(directory, 'libraries', '.imports', initial.id), initial.storageLocation);
|
||||
state.handle!.close(); state.handle = new Database(join(directory, 'controller.sqlite'));
|
||||
state.handle.pragma('foreign_keys=ON');
|
||||
expect(await browserUploadStatus(initial.id)).toMatchObject({ status: 'pending', files: [{ complete: true }] });
|
||||
const ready = await publishBrowserUpload(initial.id);
|
||||
expect(ready).toMatchObject({ id: initial.id, state: 'ready', manifestSha256: checksum, files });
|
||||
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
|
||||
});
|
||||
|
||||
it('rejects chunks racing with final publication without recreating staging files', async () => {
|
||||
const initial = begin(); await upload(initial.id, file().path, bytes);
|
||||
const publishing = publishBrowserUpload(initial.id);
|
||||
await expect(upload(initial.id, file().path, bytes)).rejects.toMatchObject({ status: 409 });
|
||||
const ready = await publishing;
|
||||
expect(ready.state).toBe('ready');
|
||||
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
|
||||
await expect(stat(join(directory, 'libraries', '.imports', initial.id))).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
});
|
||||
|
||||
it('rejects browser operations against an existing host-sourced import', async () => {
|
||||
const source = host();
|
||||
const legacy = createLibraryImport({ hostId: source.id, game: 't6', name: 'Legacy import', version: '1' });
|
||||
expect(legacy.sourceHostId).toBe(source.id);
|
||||
await expect(Promise.resolve().then(() => browserUploadStatus(legacy.id))).rejects.toThrow();
|
||||
await expect(upload(legacy.id, file().path, bytes)).rejects.toThrow();
|
||||
await expect(publishBrowserUpload(legacy.id)).rejects.toThrow();
|
||||
await expect(Promise.resolve().then(() => requestBrowserPublication(legacy.id))).rejects.toThrow();
|
||||
expect(getLibraryVersion(legacy.id)?.state).toBe('importing');
|
||||
});
|
||||
|
||||
it('distributes uploaded versions through the unchanged agent manifest, ranges, and verified cache contract', async () => {
|
||||
const initial = begin('t5'); await upload(initial.id, file().path, bytes);
|
||||
const ready = await publishBrowserUpload(initial.id);
|
||||
const target = host();
|
||||
const operation = queueLibraryDownload(target.id, ready.id);
|
||||
expect(operation).toMatchObject({ kind: 'library.download', payload: {
|
||||
libraryVersionId: ready.id, game: 't5', manifestSha256: ready.manifestSha256,
|
||||
manifestUrl: `/api/agent/libraries/${ready.id}/manifest`, filesBaseUrl: `/api/agent/libraries/${ready.id}/files`
|
||||
} });
|
||||
const manifest = manifestRoute({ request: new Request(`https://ec.test/api/agent/libraries/${ready.id}/manifest`, {
|
||||
headers: { Authorization: `Bearer ${target.credential}` }
|
||||
}), params: { id: ready.id } });
|
||||
expect(manifest.status).toBe(200);
|
||||
expect(await manifest.json()).toEqual({ id: ready.id, game: 't5', files: ready.files, sizeBytes: bytes.length, manifestSha256: ready.manifestSha256 });
|
||||
const range = await serveGameFile(new Request('https://ec.test/file', { headers: { Range: 'bytes=9-' } }), ready.id, file().path);
|
||||
expect(range.status).toBe(206);
|
||||
expect(Buffer.from(await range.arrayBuffer())).toEqual(bytes.subarray(9));
|
||||
expect(() => requireReadyLibrary(target.id, ready.id)).toThrow(/finish downloading/);
|
||||
updateHostLibraryCaches(target.id, [{ libraryVersionId: ready.id, status: 'ready', manifestSha256: '0'.repeat(64) }]);
|
||||
expect(getHostLibraryCache(target.id, ready.id)?.status).toBe('failed');
|
||||
updateHostLibraryCaches(target.id, [{ libraryVersionId: ready.id, status: 'ready', manifestSha256: ready.manifestSha256 }]);
|
||||
expect(requireReadyLibrary(target.id, ready.id).id).toBe(ready.id);
|
||||
const replacement = begin('t5', [file()], '2'); await upload(replacement.id, file().path, bytes);
|
||||
await publishBrowserUpload(replacement.id);
|
||||
expect(requireReadyLibrary(target.id, ready.id).manifestSha256).toBe(ready.manifestSha256);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
getUploadFileProgress, hashUploadedFiles, MAX_UPLOAD_CHUNK_BYTES,
|
||||
receiveUploadChunk, UploadOffsetError, validateUploadPlan, withUploadLock
|
||||
} from '../../src/lib/server/game-files/uploads.js';
|
||||
import { manifestChecksum, verifyGameFile } from '../../src/lib/server/game-files/storage.js';
|
||||
|
||||
const sha = (value: string | Uint8Array) => createHash('sha256').update(value).digest('hex');
|
||||
const stream = (value: string) => new Response(value).body!;
|
||||
const file = { path: 'zone/english/map.ff', size: 10, lastModified: 1234 };
|
||||
const chunk = (value: string, offset = 0) => ({ offset, size: Buffer.byteLength(value), sha256: sha(value) });
|
||||
let directory: string;
|
||||
let root: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
directory = await mkdtemp(join(tmpdir(), 'ec-folder-upload-'));
|
||||
root = join(directory, 'import');
|
||||
});
|
||||
afterEach(async () => { await rm(directory, { recursive: true, force: true }); });
|
||||
|
||||
describe('browser folder upload plans', () => {
|
||||
it('preserves folder structure and source identity while sorting canonical paths', () => {
|
||||
expect(validateUploadPlan([{ path: 'zone/map.ff', size: 10, lastModified: 100 }, { path: 'main/data.iwd', size: 0 }])).toEqual([
|
||||
{ path: 'main/data.iwd', size: 0, lastModified: 0 },
|
||||
{ path: 'zone/map.ff', size: 10, lastModified: 100 }
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[], [{ path: '../zone.ff', size: 1 }], [{ path: 'zone\\map.ff', size: 1 }],
|
||||
[{ path: 'Zone/a', size: 1 }, { path: 'zone/A', size: 1 }],
|
||||
[{ path: 'ZONE', size: 1 }, { path: 'zone/a', size: 1 }],
|
||||
[{ path: 'zone/a', size: -1 }], [{ path: 'zone/a', size: 1.5 }],
|
||||
[{ path: 'zone/a', size: Number.MAX_SAFE_INTEGER }, { path: 'zone/b', size: 1 }],
|
||||
[{ path: 'zone/a', size: 1, lastModified: -1 }], [{ path: 'zone/a', size: 1, lastModified: 1.5 }],
|
||||
[{ path: 'zone/a', size: 1, lastModified: null }]
|
||||
].map(plan => ({ plan })))('rejects ambiguous or unsafe plan $plan', ({ plan }) => {
|
||||
expect(() => validateUploadPlan(plan)).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('bounded resumable file transfer', () => {
|
||||
it('resumes arbitrary chunks by filesystem offset and produces a distribution-compatible manifest', async () => {
|
||||
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 0, complete: false });
|
||||
expect(await receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).toEqual({ receivedBytes: 4, complete: false });
|
||||
// A new request discovers its checkpoint from disk; no process-local receipt is required.
|
||||
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 4, complete: false });
|
||||
expect(await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'))).toEqual({ receivedBytes: 10, complete: true });
|
||||
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123456789');
|
||||
const manifest = await withUploadLock(root, () => hashUploadedFiles(root, [file]));
|
||||
expect(manifest).toEqual([{ path: file.path, size: 10, sha256: sha('0123456789') }]);
|
||||
expect(manifestChecksum(manifest)).toBe(sha(JSON.stringify(manifest)));
|
||||
await verifyGameFile(root, manifest[0]);
|
||||
expect(await readdir(root)).toEqual(['zone']);
|
||||
expect(await readdir(`${root}.chunks`)).toEqual([]);
|
||||
});
|
||||
|
||||
it('reports committed progress after reply loss instead of appending duplicate bytes', async () => {
|
||||
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
|
||||
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toMatchObject({
|
||||
name: 'UploadOffsetError', receivedBytes: 4, status: 409
|
||||
});
|
||||
await expect(receiveUploadChunk(root, file, chunk('89', 8), stream('89'))).rejects.toBeInstanceOf(UploadOffsetError);
|
||||
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
|
||||
await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'));
|
||||
});
|
||||
|
||||
it('can resume a verified append interrupted by process termination at an unaligned byte', async () => {
|
||||
await mkdir(join(root, 'zone/english'), { recursive: true });
|
||||
await writeFile(join(root, file.path), '0123456');
|
||||
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 7, complete: false });
|
||||
await receiveUploadChunk(root, file, chunk('789', 7), stream('789'));
|
||||
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123456789');
|
||||
});
|
||||
|
||||
it('distinguishes an absent empty file and creates it using the empty digest', async () => {
|
||||
const empty = { path: 'main/empty.dat', size: 0 };
|
||||
expect(await getUploadFileProgress(root, empty)).toEqual({ receivedBytes: 0, complete: false });
|
||||
await expect(hashUploadedFiles(root, [empty])).rejects.toThrow();
|
||||
expect(await receiveUploadChunk(root, empty, chunk(''), null)).toEqual({ receivedBytes: 0, complete: true });
|
||||
expect(await getUploadFileProgress(root, empty)).toEqual({ receivedBytes: 0, complete: true });
|
||||
expect(await hashUploadedFiles(root, [empty])).toEqual([{ ...empty, sha256: sha('') }]);
|
||||
});
|
||||
|
||||
it('does not advance progress for wrong checksums, truncated requests, oversized streams or disconnects', async () => {
|
||||
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
|
||||
await expect(receiveUploadChunk(root, file, chunk('456789', 4), stream('xxxxxx'))).rejects.toThrow(/checksum/);
|
||||
await expect(receiveUploadChunk(root, file, chunk('456789', 4), stream('45'))).rejects.toThrow(/size mismatch/);
|
||||
await expect(receiveUploadChunk(root, file, chunk('456789', 4), stream('4567890'))).rejects.toThrow(/exceeds/);
|
||||
let reads = 0;
|
||||
const disconnect = new ReadableStream<Uint8Array>({ pull(controller) {
|
||||
if (reads++ === 0) controller.enqueue(new TextEncoder().encode('45'));
|
||||
else controller.error(new Error('Browser connection ended.'));
|
||||
} });
|
||||
await expect(receiveUploadChunk(root, file, chunk('456789', 4), disconnect)).rejects.toThrow(/connection ended/);
|
||||
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 4, complete: false });
|
||||
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
|
||||
expect(await readdir(`${root}.chunks`)).toEqual([]);
|
||||
await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'));
|
||||
});
|
||||
|
||||
it('bounds request size before reading any body and rejects invalid metadata', async () => {
|
||||
const tooLarge = MAX_UPLOAD_CHUNK_BYTES + 1;
|
||||
await expect(receiveUploadChunk(root, { path: 'large.iwd', size: tooLarge }, {
|
||||
offset: 0, size: tooLarge, sha256: sha('')
|
||||
}, null)).rejects.toThrow(/Invalid upload chunk/);
|
||||
for (const metadata of [
|
||||
{ ...chunk('a'), offset: -1 }, { ...chunk('a'), offset: 0.5 }, { ...chunk('a'), size: 1.5 },
|
||||
{ ...chunk('a'), sha256: 'wrong' }, chunk('', 0), chunk('01234567890', 0)
|
||||
]) await expect(receiveUploadChunk(root, file, metadata, stream('a'))).rejects.toThrow(/Invalid upload chunk/);
|
||||
await expect(receiveUploadChunk(root, file, chunk('a'), null)).rejects.toThrow(/body is required/);
|
||||
});
|
||||
|
||||
it('streams a maximum-size chunk supplied in small pieces', async () => {
|
||||
const piece = new Uint8Array(64 * 1024).fill(97);
|
||||
let remaining = MAX_UPLOAD_CHUNK_BYTES;
|
||||
const digest = createHash('sha256');
|
||||
for (let i = 0; i < MAX_UPLOAD_CHUNK_BYTES / piece.length; i++) digest.update(piece);
|
||||
const body = new ReadableStream<Uint8Array>({ pull(controller) {
|
||||
if (remaining === 0) controller.close();
|
||||
else { remaining -= piece.length; controller.enqueue(piece); }
|
||||
} });
|
||||
const large = { path: 'main/large.iwd', size: MAX_UPLOAD_CHUNK_BYTES };
|
||||
const sha256 = digest.digest('hex');
|
||||
await receiveUploadChunk(root, large, { offset: 0, size: large.size, sha256 }, body);
|
||||
expect(await hashUploadedFiles(root, [large])).toEqual([{ ...large, sha256 }]);
|
||||
});
|
||||
|
||||
it('rejects missing or incorrectly sized files before publication', async () => {
|
||||
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
|
||||
await expect(hashUploadedFiles(root, [file])).rejects.toThrow(/Size verification/);
|
||||
await writeFile(join(root, file.path), '01234567890');
|
||||
await expect(getUploadFileProgress(root, file)).rejects.toThrow(/exceeds/);
|
||||
});
|
||||
|
||||
it('refuses traversal and symlinks in destination and temporary storage', async () => {
|
||||
await mkdir(root);
|
||||
const outside = join(directory, 'outside');
|
||||
await mkdir(outside);
|
||||
await expect(receiveUploadChunk(root, { path: '../escape', size: 1 }, chunk('a'), stream('a'))).rejects.toThrow(/relative/);
|
||||
await symlink(outside, join(root, 'zone'));
|
||||
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toThrow(/non-regular/);
|
||||
await rm(join(root, 'zone'));
|
||||
await symlink(outside, `${root}.chunks`);
|
||||
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toThrow(/non-regular/);
|
||||
expect(await readdir(outside)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('upload and publication coordination', () => {
|
||||
it('serializes simultaneous writes with the same offset', async () => {
|
||||
const responses = await Promise.allSettled([
|
||||
receiveUploadChunk(root, file, chunk('0123'), stream('0123')),
|
||||
receiveUploadChunk(root, file, chunk('0123'), stream('0123'))
|
||||
]);
|
||||
expect(responses.map(response => response.status)).toEqual(['fulfilled', 'rejected']);
|
||||
expect((responses[1] as PromiseRejectedResult).reason).toMatchObject({ name: 'UploadOffsetError', receivedBytes: 4 });
|
||||
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
|
||||
});
|
||||
|
||||
it('rechecks writable state inside the lock after queued publication and releases failed locks', async () => {
|
||||
let finishPublication!: () => void;
|
||||
const gate = new Promise<void>(resolve => { finishPublication = resolve; });
|
||||
let published = false;
|
||||
const publication = withUploadLock(root, async () => { await gate; published = true; });
|
||||
const transfer = receiveUploadChunk(root, file, chunk('0123'), stream('0123'), () => {
|
||||
if (published) throw new Error('Library is immutable.');
|
||||
});
|
||||
finishPublication();
|
||||
await publication;
|
||||
await expect(transfer).rejects.toThrow(/immutable/);
|
||||
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 0, complete: false });
|
||||
await expect(withUploadLock(root, () => { throw new Error('Failed work'); })).rejects.toThrow(/Failed work/);
|
||||
expect(await withUploadLock(root, () => 'next operation')).toBe('next operation');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,75 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { selectGameFolder } from '../../src/lib/ui/game-folder.js';
|
||||
|
||||
function selectedFile(path: string, size = 10) {
|
||||
const read = vi.fn(() => { throw new Error('Folder selection must not read file bodies.'); });
|
||||
const file = {
|
||||
name: path.split('/').at(-1), webkitRelativePath: path, size, lastModified: 1_700_000_000_000,
|
||||
arrayBuffer: read, text: read, stream: read, slice: read
|
||||
} as unknown as File;
|
||||
return { file, read };
|
||||
}
|
||||
|
||||
describe('game folder selection', () => {
|
||||
it('removes the selected root exactly once while preserving nested game paths', () => {
|
||||
const first = selectedFile('T6/T6/zone/english/base.ff', 20);
|
||||
const second = selectedFile('T6/main/iw_00.iwd', 30);
|
||||
const result = selectGameFolder([first.file, second.file]);
|
||||
expect(result.name).toBe('T6');
|
||||
expect(result.size).toBe(50);
|
||||
expect(result.files).toEqual([
|
||||
{ path: 'T6/zone/english/base.ff', file: first.file },
|
||||
{ path: 'main/iw_00.iwd', file: second.file }
|
||||
]);
|
||||
expect(first.read).not.toHaveBeenCalled();
|
||||
expect(second.read).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('prepares multi-gigabyte and empty files using metadata alone', () => {
|
||||
const large = selectedFile('Game/zone/base.ff', 7 * 1024 ** 3);
|
||||
const empty = selectedFile('Game/empty.txt', 0);
|
||||
const result = selectGameFolder([large.file, empty.file]);
|
||||
expect(result.size).toBe(7 * 1024 ** 3);
|
||||
expect(result.files.map(item => item.path)).toEqual(['empty.txt', 'zone/base.ff']);
|
||||
expect(result.files[1].file).toBe(large.file);
|
||||
expect(large.read).not.toHaveBeenCalled();
|
||||
expect(empty.read).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects case-insensitive duplicate file names', () => {
|
||||
expect(() => selectGameFolder([
|
||||
selectedFile('Game/Zone/base.ff').file,
|
||||
selectedFile('Game/zone/BASE.ff').file
|
||||
])).toThrow('duplicate game paths');
|
||||
});
|
||||
|
||||
it('rejects a path used as both a file and a directory regardless of case', () => {
|
||||
expect(() => selectGameFolder([
|
||||
selectedFile('Game/zone/english/base.ff').file,
|
||||
selectedFile('Game/ZONE').file
|
||||
])).toThrow('same game path for a file and a directory');
|
||||
});
|
||||
|
||||
it.each(['../base.ff', './base.ff', '/Game/base.ff', 'base.ff'])(
|
||||
'rejects an absent or parent root in %s', path => {
|
||||
expect(() => selectGameFolder([selectedFile(path).file])).toThrow('one game folder');
|
||||
}
|
||||
);
|
||||
|
||||
it.each(['Game/../base.ff', 'Game/zone/./base.ff', 'Game/zone//base.ff', 'Game/zone\\base.ff'])(
|
||||
'rejects traversal and non-normalized content paths in %s', path => {
|
||||
expect(() => selectGameFolder([selectedFile(path).file])).toThrow('unsupported game path');
|
||||
}
|
||||
);
|
||||
|
||||
it('rejects files from multiple selected roots', () => {
|
||||
expect(() => selectGameFolder([
|
||||
selectedFile('T5/main/base.ff').file,
|
||||
selectedFile('T6/main/base.ff').file
|
||||
])).toThrow('one game folder');
|
||||
});
|
||||
|
||||
it('rejects empty selections', () => {
|
||||
expect(() => selectGameFolder([])).toThrow('contains no files');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,263 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import Database from 'better-sqlite3';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
let fixture: Fixture;
|
||||
const inventory = { hostname: 'games-1', protocolVersion: 1, capabilities: ['host.decommission'], dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3 };
|
||||
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
||||
afterEach(async () => { vi.unstubAllEnvs(); await fixture.cleanup(); });
|
||||
|
||||
async function enrolled() {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const host = hosts.createHost({ name: 'Game host' });
|
||||
const credentials = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
|
||||
return { ...hosts, host, credentials };
|
||||
}
|
||||
|
||||
async function finishCleanup(hosts: Awaited<ReturnType<typeof enrolled>>) {
|
||||
const result = hosts.deleteHost(hosts.host.id);
|
||||
const operation = hosts.getHostOperation(result.operationId!)!;
|
||||
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
|
||||
const token = operation.payload.receiptToken as string;
|
||||
hosts.receiveDecommissionReceipt(operation.id, token, { stage: 'cleaned' });
|
||||
hosts.receiveDecommissionReceipt(operation.id, token, { stage: 'completed' });
|
||||
return operation;
|
||||
}
|
||||
|
||||
async function instance(hostId: string, retired = false) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at,retired_at)
|
||||
VALUES('instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?,?)`)
|
||||
.run(hostId, new Date().toISOString(), retired ? new Date().toISOString() : null);
|
||||
}
|
||||
|
||||
describe('host deletion', () => {
|
||||
it('adds the deletion marker to existing host schemas without losing hosts', async () => {
|
||||
const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js');
|
||||
const legacy = new Database(':memory:');
|
||||
try {
|
||||
legacy.exec("CREATE TABLE ec_hosts(id TEXT PRIMARY KEY, name TEXT NOT NULL); INSERT INTO ec_hosts VALUES('old','Existing host')");
|
||||
ensureHostSchema(legacy); ensureHostSchema(legacy);
|
||||
expect(legacy.prepare('SELECT * FROM ec_hosts').get()).toEqual({ id: 'old', name: 'Existing host', deleted_at: null });
|
||||
} finally { legacy.close(); }
|
||||
});
|
||||
|
||||
it('deletes an unenrolled host and invalidates its unused enrollment', async () => {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const host = hosts.createHost({ name: 'Not installed' });
|
||||
const enrollment = hosts.createEnrollment(host.id);
|
||||
expect(hosts.hostDeletionStatus(host.id)).toEqual({ allowed: true, blockers: [] });
|
||||
hosts.deleteHost(host.id);
|
||||
expect(hosts.getHost(host.id)).toBeNull(); expect(hosts.listHosts()).toEqual([]);
|
||||
expect(() => hosts.enrollHost(enrollment.token, inventory)).toThrow('invalid or expired');
|
||||
expect(() => hosts.createEnrollment(host.id)).toThrow('not found');
|
||||
expect(() => hosts.deleteHost(host.id)).toThrow(expect.objectContaining({ status: 404 }));
|
||||
});
|
||||
|
||||
it('waits for host cleanup before revoking credentials and retains controller libraries and history', async () => {
|
||||
const hosts = await enrolled();
|
||||
const other = hosts.createHost({ name: 'Keep' });
|
||||
const otherCredentials = hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory);
|
||||
const enrollment = hosts.createEnrollment(hosts.host.id);
|
||||
const completed = hosts.queueHostOperation(hosts.host.id, 'host.inspect', {});
|
||||
hosts.reportHostOperation(hosts.host.id, completed.id, { state: 'completed', result: { hostname: 'games-1' } });
|
||||
await instance(hosts.host.id, true);
|
||||
const file = join(fixture.root, 'game-files-sentinel');
|
||||
await writeFile(file, 'Game files remain');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { ensureGameFileSchema } = await import('../../src/lib/server/game-files/schema.js');
|
||||
ensureGameFileSchema(db());
|
||||
db().prepare("INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES('library','T5','t5','',?)").run(new Date().toISOString());
|
||||
db().prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume)
|
||||
VALUES('version','library','1','','[]','ready',?,?,?,'t5_game_files')`).run(new Date().toISOString(), file, hosts.host.id);
|
||||
db().prepare("INSERT INTO host_library_caches(host_id,version_id,status,updated_at) VALUES(?,'version','ready',?)").run(hosts.host.id, new Date().toISOString());
|
||||
|
||||
const queued = hosts.deleteHost(hosts.host.id);
|
||||
expect(queued.deleted).toBe(false);
|
||||
expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('queued');
|
||||
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
|
||||
await finishCleanup(hosts);
|
||||
|
||||
expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull();
|
||||
expect(hosts.authenticateCredential(otherCredentials.credential)?.id).toBe(other.id);
|
||||
expect(() => hosts.enrollHost(enrollment.token, inventory)).toThrow('invalid or expired');
|
||||
expect(hosts.listHosts().map(host => host.id)).toEqual([other.id]);
|
||||
expect(hosts.getHostOperation(completed.id)).toMatchObject({ state: 'completed', result: { hostname: 'games-1' } });
|
||||
expect(hosts.listHostOperations(hosts.host.id)).toHaveLength(2);
|
||||
expect(db().prepare('SELECT mode,deleted_at FROM ec_hosts WHERE id=?').get(hosts.host.id)).toEqual({ mode: 'disabled', deleted_at: expect.any(String) });
|
||||
expect(db().prepare('SELECT revoked_at FROM ec_host_credentials WHERE host_id=?').get(hosts.host.id)).toEqual({ revoked_at: expect.any(String) });
|
||||
expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({ count: 1 });
|
||||
expect(db().prepare('SELECT COUNT(*) AS count FROM host_library_caches').get()).toEqual({ count: 1 });
|
||||
expect(db().prepare("SELECT state,source_host_id FROM game_file_versions WHERE id='version'").get()).toEqual({ state: 'ready', source_host_id: hosts.host.id });
|
||||
expect(await readFile(file, 'utf8')).toBe('Game files remain');
|
||||
expect(() => hosts.heartbeat(hosts.host.id, inventory)).toThrow('not found');
|
||||
expect(() => hosts.rotateCredential(hosts.host.id)).toThrow('not found');
|
||||
expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow('not found');
|
||||
expect(() => hosts.reportHostOperation(hosts.host.id, completed.id, { state: 'running' })).toThrow('not found');
|
||||
expect(() => hosts.updateHost(hosts.host.id, { mode: 'active' })).toThrow('not found');
|
||||
});
|
||||
|
||||
it('cancels only undelivered queued operations and retains their journal', async () => {
|
||||
const hosts = await enrolled();
|
||||
const operations = ['runtime.pull', 'library.download', 'credentials.rotate'].map(kind => hosts.queueHostOperation(hosts.host.id, kind, {}));
|
||||
expect(hosts.hostDeletionStatus(hosts.host.id).allowed).toBe(true);
|
||||
hosts.deleteHost(hosts.host.id);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
for (const operation of operations) {
|
||||
expect(hosts.getHostOperation(operation.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('before this operation was delivered') });
|
||||
expect(db().prepare('SELECT state FROM ec_host_operation_events WHERE operation_id=? ORDER BY id').all(operation.id)).toEqual([{ state: 'pending' }, { state: 'failed' }]);
|
||||
}
|
||||
});
|
||||
|
||||
it('blocks live instances even when no server record or port reservation exists', async () => {
|
||||
const hosts = await enrolled();
|
||||
await instance(hosts.host.id);
|
||||
const status = hosts.hostDeletionStatus(hosts.host.id);
|
||||
expect(status.allowed).toBe(false); expect(status.blockers.join(' ')).toContain('t5_zombie_theater_1');
|
||||
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
|
||||
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
|
||||
});
|
||||
|
||||
it('blocks residual port reservations independently of live instances', async () => {
|
||||
const hosts = await enrolled();
|
||||
hosts.reserveHostPort(hosts.host.id, 'unresolved-instance');
|
||||
expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('reserved game port')] });
|
||||
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
|
||||
hosts.releaseHostPorts('unresolved-instance');
|
||||
await finishCleanup(hosts);
|
||||
expect(hosts.getHost(hosts.host.id)).toBeNull();
|
||||
});
|
||||
|
||||
it.each(['pending', 'accepted', 'running', 'unknown'] as const)('blocks unresolved %s work after delivery without revoking access or cancelling other work', async state => {
|
||||
const hosts = await enrolled();
|
||||
const operation = hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {});
|
||||
hosts.pendingHostOperations(hosts.host.id);
|
||||
if (state !== 'pending') hosts.reportHostOperation(hosts.host.id, operation.id, { state });
|
||||
const queued = hosts.queueHostOperation(hosts.host.id, 'library.download', {});
|
||||
hosts.updateHost(hosts.host.id, { mode: 'disabled' });
|
||||
expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('delivered operation')] });
|
||||
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
|
||||
expect(hosts.getHostOperation(queued.id)?.state).toBe('pending');
|
||||
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
|
||||
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' });
|
||||
hosts.deleteHost(hosts.host.id);
|
||||
expect(hosts.getHostOperation(queued.id)?.state).toBe('failed');
|
||||
});
|
||||
|
||||
it('rechecks blockers on DELETE even after a successful eligibility check', async () => {
|
||||
const hosts = await enrolled();
|
||||
const route = await import('../../src/routes/api/hosts/[id]/+server.js');
|
||||
const url = `https://ec.test/api/hosts/${hosts.host.id}`;
|
||||
const params = { id: hosts.host.id };
|
||||
expect(await route.GET({ request: new Request(url), params }).json()).toMatchObject({ deletion: { allowed: true, blockers: [] } });
|
||||
const operation = hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {});
|
||||
hosts.pendingHostOperations(hosts.host.id);
|
||||
const blocked = route.DELETE({ request: new Request(url, { method: 'DELETE' }), params });
|
||||
expect(blocked.status).toBe(409);
|
||||
expect(await blocked.json()).toMatchObject({ error: expect.any(String), blockers: [expect.stringContaining('delivered operation')] });
|
||||
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' });
|
||||
const removed = route.DELETE({ request: new Request(url, { method: 'DELETE' }), params });
|
||||
expect(removed.status).toBe(202); expect(await removed.json()).toMatchObject({ deleted: false, operationId: expect.any(String) });
|
||||
await finishCleanup(hosts);
|
||||
expect(route.DELETE({ request: new Request(url, { method: 'DELETE' }), params }).status).toBe(404);
|
||||
});
|
||||
|
||||
it('keeps DELETE restricted to admin sessions and same-origin management requests', async () => {
|
||||
const hosts = await enrolled();
|
||||
const { routePermission, roleAllows } = await import('../../src/lib/server/auth/access.js');
|
||||
const permission = routePermission('/api/hosts/[id]', 'DELETE');
|
||||
expect(roleAllows('admin', permission)).toBe(true);
|
||||
expect(roleAllows('operator', permission)).toBe(false); expect(roleAllows('viewer', permission)).toBe(false);
|
||||
const { DELETE } = await import('../../src/routes/api/hosts/[id]/+server.js');
|
||||
for (const headers of [new Headers({ origin: 'https://attacker.test' }), new Headers({ authorization: `Bearer ${hosts.credentials.credential}` })]) {
|
||||
expect(() => DELETE({ request: new Request('https://ec.test/api/hosts/id', { method: 'DELETE', headers }), params: { id: hosts.host.id } })).toThrow(expect.objectContaining({ status: 403 }));
|
||||
}
|
||||
expect(hosts.getHost(hosts.host.id)).not.toBeNull();
|
||||
});
|
||||
|
||||
it('requires an uninstall-capable agent and preserves enrollment access for upgrades', async () => {
|
||||
const hosts = await enrolled();
|
||||
hosts.heartbeat(hosts.host.id, { ...inventory, capabilities: [] });
|
||||
expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('Update this host agent')] });
|
||||
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
|
||||
expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy();
|
||||
expect(hosts.getHost(hosts.host.id)?.mode).toBe('active');
|
||||
});
|
||||
|
||||
it('keeps offline cleanup pending and blocks settings, new work, enrollment and credential revocation', async () => {
|
||||
const hosts = await enrolled();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2020-01-01T00:00:00Z', hosts.host.id);
|
||||
const queued = hosts.deleteHost(hosts.host.id);
|
||||
expect(hosts.deleteHost(hosts.host.id)).toEqual(queued);
|
||||
expect(hosts.getHost(hosts.host.id)).toMatchObject({ connected: false, readiness: 'deleting', decommission: { stage: 'queued' } });
|
||||
expect(hosts.pendingHostOperations(hosts.host.id)).toEqual([]);
|
||||
expect(() => hosts.updateHost(hosts.host.id, { mode: 'disabled' })).toThrow('cleanup');
|
||||
expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow('cleanup');
|
||||
expect(() => hosts.reserveHostPort(hosts.host.id, 'late-instance')).toThrow('cleanup');
|
||||
expect(hosts.hostPortAvailable(hosts.host.id, hosts.host.portMin)).toBe(false);
|
||||
expect(() => hosts.createEnrollment(hosts.host.id)).toThrow('cleanup');
|
||||
expect(() => hosts.revokeCredentials(hosts.host.id)).toThrow('cleanup');
|
||||
hosts.heartbeat(hosts.host.id, inventory);
|
||||
expect(hosts.pendingHostOperations(hosts.host.id).map(operation => operation.id)).toEqual([queued.operationId]);
|
||||
});
|
||||
|
||||
it('requires scoped receipts in order and never treats an ordinary completion report or loss of contact as removal', async () => {
|
||||
const hosts = await enrolled();
|
||||
const queued = hosts.deleteHost(hosts.host.id);
|
||||
const operation = hosts.getHostOperation(queued.operationId!)!;
|
||||
const token = operation.payload.receiptToken as string;
|
||||
const receipt = (stage: string, credential = token) => hosts.receiveDecommissionReceipt(operation.id, credential, { stage });
|
||||
expect(() => receipt('cleaned')).toThrow(expect.objectContaining({ status: 409 }));
|
||||
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
|
||||
expect(() => receipt('cleaned', hosts.credentials.credential)).toThrow(expect.objectContaining({ status: 401 }));
|
||||
expect(() => receipt('completed')).toThrow(expect.objectContaining({ status: 409 }));
|
||||
expect(() => hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' })).toThrow('cleanup worker');
|
||||
receipt('cleaned');
|
||||
expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('cleaned');
|
||||
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
|
||||
const route = await import('../../src/routes/api/hosts/[id]/+server.js');
|
||||
const detail = await route.GET({ request: new Request('https://ec.test/api/hosts/' + hosts.host.id), params: { id: hosts.host.id } }).text();
|
||||
expect(detail).not.toContain(token);
|
||||
receipt('completed');
|
||||
expect(hosts.getHost(hosts.host.id)).toBeNull();
|
||||
expect(receipt('completed')).toEqual({ acknowledged: true });
|
||||
expect(receipt('cleaned')).toEqual({ acknowledged: true });
|
||||
expect(hosts.getHostOperation(operation.id)?.state).toBe('completed');
|
||||
expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull();
|
||||
});
|
||||
|
||||
it('retains cleanup failures and retries only work rejected before mutation with a new scoped receipt', async () => {
|
||||
const hosts = await enrolled();
|
||||
const first = hosts.deleteHost(hosts.host.id);
|
||||
hosts.reportHostOperation(hosts.host.id, first.operationId!, { state: 'failed', error: 'Ownership marker requires repair' });
|
||||
expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ stage: 'failed', retryable: true });
|
||||
expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy();
|
||||
const second = hosts.deleteHost(hosts.host.id);
|
||||
expect(second.operationId).not.toBe(first.operationId);
|
||||
const operation = hosts.getHostOperation(second.operationId!)!;
|
||||
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
|
||||
hosts.receiveDecommissionReceipt(operation.id, operation.payload.receiptToken as string, { stage: 'failed', error: 'Filesystem cleanup failed; retrying' });
|
||||
expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ stage: 'failed', retryable: false, error: 'Filesystem cleanup failed; retrying' });
|
||||
expect(hosts.deleteHost(hosts.host.id)).toEqual(second);
|
||||
await finishCleanup(hosts);
|
||||
expect(hosts.getHostOperation(first.operationId!)?.state).toBe('failed');
|
||||
});
|
||||
|
||||
it('authenticates cleanup receipts independently after the agent credential has been revoked', async () => {
|
||||
const hosts = await enrolled();
|
||||
const queued = hosts.deleteHost(hosts.host.id);
|
||||
const operation = hosts.getHostOperation(queued.operationId!)!;
|
||||
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
|
||||
const { POST } = await import('../../src/routes/api/agent/decommission/[id]/+server.js');
|
||||
const send = (stage: string, token: string) => POST({ request: new Request(`https://ec.test/api/agent/decommission/${operation.id}`, {
|
||||
method: 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ stage })
|
||||
}), params: { id: operation.id } });
|
||||
expect((await send('cleaned', hosts.credentials.credential)).status).toBe(401);
|
||||
expect((await send('cleaned', operation.payload.receiptToken as string)).status).toBe(200);
|
||||
expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(200);
|
||||
expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(200);
|
||||
});
|
||||
});
|
||||
@@ -71,7 +71,7 @@ describe('native host readiness and reservations', () => {
|
||||
expect(api.requireReadyHost(api.host.id).readiness).toBe('ready');
|
||||
expect(() => api.requireReadyHost(api.host.id, { image: 'missing:image' })).toThrow('runtime');
|
||||
expect(() => api.requireReadyHost(api.host.id, { libraryId: 'missing' })).toThrow('library');
|
||||
expect(api.heartbeat(api.host.id, { ...inventory, libraries: [] }).readiness).toBe('preparing');
|
||||
expect(api.heartbeat(api.host.id, { ...inventory, libraries: [] }).readiness).toBe('needs-game-files');
|
||||
const host = api.heartbeat(api.host.id, { ...inventory, dockerRunning: false });
|
||||
expect(host.connected).toBe(true); expect(host.readiness).toBe('degraded');
|
||||
expect(() => api.requireReadyHost(host.id)).toThrow('Docker');
|
||||
@@ -79,6 +79,40 @@ describe('native host readiness and reservations', () => {
|
||||
vi.useFakeTimers(); vi.setSystemTime(Date.now() + 91_000);
|
||||
expect(api.getHost(host.id)?.connectionState).toBe('offline'); expect(() => api.requireReadyHost(host.id)).toThrow('stale');
|
||||
});
|
||||
it.each([
|
||||
{ missing: { libraries: [] }, expected: 'needs-game-files' },
|
||||
{ missing: { runtimeImages: [] }, expected: 'needs-runtime' },
|
||||
{ missing: { runtimeImages: [], libraries: [] }, expected: 'needs-runtime' }
|
||||
])('names missing content as $expected until a prepared heartbeat arrives', async ({ missing, expected }) => {
|
||||
const api = await enrolled();
|
||||
const host = api.heartbeat(api.host.id, { ...inventory, ...missing });
|
||||
expect(host.connected).toBe(true);
|
||||
expect(host.readiness).toBe(expected);
|
||||
expect(() => api.requireReadyHost(host.id)).toThrow();
|
||||
expect(api.heartbeat(host.id, inventory).readiness).toBe('ready');
|
||||
expect(api.requireReadyHost(host.id).id).toBe(host.id);
|
||||
});
|
||||
it('keeps setup work deliverable while new instance creation waits for missing game files', async () => {
|
||||
const api = await enrolled();
|
||||
const create = api.queueHostOperation(api.host.id, 'instance.create', {});
|
||||
api.heartbeat(api.host.id, { ...inventory, libraries: [] });
|
||||
expect(() => api.queueHostOperation(api.host.id, 'instance.create', {})).toThrow('game-file library');
|
||||
const download = api.queueHostOperation(api.host.id, 'library.download', {});
|
||||
const runtime = api.queueHostOperation(api.host.id, 'runtime.pull', {});
|
||||
expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toEqual([download.id, runtime.id]);
|
||||
api.heartbeat(api.host.id, { ...inventory, runtimeImages: [], libraries: [] });
|
||||
expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toEqual([download.id, runtime.id]);
|
||||
api.heartbeat(api.host.id, inventory);
|
||||
expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toContain(create.id);
|
||||
});
|
||||
it('gives enrollment, infrastructure failure and host modes priority over missing content', async () => {
|
||||
const api = await enrolled();
|
||||
expect(api.createHost({ name: 'New host' }).readiness).toBe('preparing');
|
||||
expect(api.heartbeat(api.host.id, { ...inventory, libraries: [], dockerRunning: false }).readiness).toBe('degraded');
|
||||
api.heartbeat(api.host.id, { ...inventory, libraries: [] });
|
||||
expect(api.updateHost(api.host.id, { mode: 'draining' }).readiness).toBe('draining');
|
||||
expect(api.updateHost(api.host.id, { mode: 'disabled' }).readiness).toBe('disabled');
|
||||
});
|
||||
it('blocks new deployments on a draining or disabled host without altering existing operations', async () => {
|
||||
const api = await enrolled(); const existing = api.queueHostOperation(api.host.id, 'instance.inspect', {}, { instanceId: 'immutable-id', generation: 1 });
|
||||
api.updateHost(api.host.id, { mode: 'draining' });
|
||||
|
||||
Reference in New Issue
Block a user