Compare commits

...
4 Commits
40 changed files with 3214 additions and 158 deletions
+12 -7
View File
@@ -29,11 +29,13 @@ The default Compose data mount preserves the previous
`EC_DATA_DIR` to your preferred persistent directory. Back up that directory,
including SQLite metadata and published game-file storage.
1. **Hosts → Add Host**: configure a Debian 12/13 amd64 machine and run its
1. **Library → Game Files**: select T5 or T6 and upload the complete local game
folder, including its DLC files. Name the library and version, then publish it.
Uploads work before any host is enrolled.
2. **Hosts → Add Host**: configure a Debian 12/13 amd64 machine and run its
generated enrollment command.
2. Prepare a Plutainer runtime from the host page.
3. **Library → Game Files**: import existing T5/T6 files once, then cache a
published version on the destination host.
3. Prepare a Plutainer runtime from the host page and download the published
game-file version to that host from **Game Files**.
4. Add the exact Plutonium server keys in **Keys**, or supply missing keys when
launching.
5. Launch a server or collection with an explicit host, game-file version and
@@ -69,9 +71,12 @@ server ID and retained after removal.
## Libraries and campaigns
Game-file versions include game, version, DLC metadata, manifests, checksums and
size. Actual game files use dedicated filesystem storage, never SQLite blobs.
Downloads resume, report progress and verify checksums before becoming usable.
Game-file versions contain the uploaded T5/T6 folder, with its directory layout
and included DLC preserved. There is no separate DLC declaration. Versions record
the game, label, file manifest, checksums and size. Uploads use bounded chunks,
support pause/resume and verify checksums before publication. Actual game files
use dedicated filesystem storage, never SQLite blobs. Host downloads resume,
report progress and verify checksums before becoming usable.
Multiple instances on one host share read-only base files.
Scripts, plugins, configuration baselines and overlays retain their versioning,
+338 -4
View File
@@ -29,10 +29,17 @@ import urllib.parse
import urllib.request
import uuid
VERSION = "1.0.0"
VERSION = "1.1.0"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service")
UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service")
UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
ROOT_MARKER = ".electric-cherry.json"
class AgentError(Exception):
@@ -47,6 +54,97 @@ class Rejected(AgentError):
"""A definitive precondition failure, before any destructive action."""
def systemctl(*args):
try:
result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
except (OSError, subprocess.TimeoutExpired):
raise AgentError("Host service management is unavailable") from None
if result.returncode:
raise AgentError("Host service management did not complete")
def protected_path(path, missing=False):
"""Deletion authority must never come from a runtime-writable path."""
path = Path(path)
for current in (*reversed(path.parents), path):
try:
info = current.lstat()
except FileNotFoundError:
if missing:
continue
raise AgentError("Host uninstall files are unavailable; reinstall the agent") from None
if stat_module.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
raise AgentError("Host uninstall paths must be owned and writable only by root")
return path
def protected_file(path, missing=False):
path = protected_path(path, missing=missing)
try:
if not stat_module.S_ISREG(path.lstat().st_mode):
raise AgentError("Host uninstall metadata must be a regular file")
except FileNotFoundError:
if not missing:
raise
return path
def sync_parent(path):
descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
def uninstall_root(value, identity=None):
"""Accept only a canonical, dedicated directory with no nested mounts."""
if not isinstance(value, str) or not re.fullmatch(r"/[A-Za-z0-9/_.-]+", value):
raise AgentError("Host data directory is unsafe to remove")
root = Path(value)
forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"}
if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden:
raise AgentError("Host data directory is unsafe to remove")
for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT):
if installed == root or root in installed.parents:
raise AgentError("Host data directory overlaps the agent installation")
protected_path(root, missing=True)
if root.exists():
info = root.stat()
if not stat_module.S_ISDIR(info.st_mode):
raise AgentError("Host data directory is not a directory")
if identity and [info.st_dev, info.st_ino] != identity:
raise AgentError("Host data directory changed since deletion was authorized")
try:
# ProtectSystem/ReadWritePaths give the running agent a private bind
# mount at its data directory. Inspect the host systemd namespace,
# otherwise every legitimate installation appears to be a mount root.
mount_lines = Path("/proc/1/mountinfo").read_text().splitlines()
except FileNotFoundError:
mount_lines = [] # Non-Linux tests; the installed service requires Linux.
for line in mount_lines:
fields = line.split()
if len(fields) < 5:
continue
mount = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), fields[4])
if mount == value or mount.startswith(value + "/"):
raise AgentError("Unmount filesystems inside the host data directory before deletion")
return root
def uninstall_ownership(config):
root = uninstall_root(config.get("root"))
marker = protected_file(root / ROOT_MARKER)
try:
saved = json.loads(marker.read_text())
except (OSError, ValueError):
raise AgentError("Host data ownership is unavailable; reinstall the agent") from None
expected = {"version": 1, "root": str(root), "hostId": config.get("hostId"), "controller": config.get("controller", "").rstrip("/")}
if saved != expected or not expected["hostId"]:
raise AgentError("Host data ownership does not match this enrollment")
return root
def digest(data):
return hashlib.sha256(data).hexdigest()
@@ -275,7 +373,7 @@ class Docker:
raise AgentError("Docker command unavailable or timed out") from None
if result.returncode:
# Never include arbitrary Docker stderr: create errors may echo secrets.
if tolerate_missing and (b"No such object" in (result.stderr or b"") or b"No such container" in (result.stderr or b"")):
if tolerate_missing and any(message in (result.stderr or b"") for message in (b"No such object", b"No such container", b"No such image")):
return None
raise AgentError("Docker rejected the requested operation")
return result.stdout.decode("utf-8", "replace")
@@ -285,6 +383,188 @@ class Docker:
return json.loads(raw)[0] if raw is not None else None
class Uninstaller:
"""A separate systemd service owns cleanup after stopping the host agent.
Its root-only request and executable stay on disk until the controller
acknowledges data removal. Completion is reported only after removing those
final files. A lost final response can be retried from memory; a power loss
in that last window deliberately leaves controller status unconfirmed.
"""
def __init__(self, request_path, docker=None):
if Path(request_path) != UNINSTALL_REQUEST:
raise AgentError("Invalid host uninstall request location")
protected_file(UNINSTALL_REQUEST)
self.job = json.loads(UNINSTALL_REQUEST.read_text())
self.docker = docker or Docker()
self.root = uninstall_root(self.job.get("root"), self.job.get("rootIdentity"))
host_id = identifier(self.job.get("hostId"))
operation_id = identifier(self.job.get("operationId"))
parsed = urllib.parse.urlsplit(self.job.get("controller", ""))
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment:
raise AgentError("Host uninstall controller must use HTTPS")
if self.job.get("receiptUrl") != "/api/agent/decommission/" + operation_id or not isinstance(self.job.get("receiptToken"), str) or not self.job["receiptToken"]:
raise AgentError("Invalid host uninstall receipt")
if not isinstance(self.job.get("rootIdentity"), list) or len(self.job["rootIdentity"]) != 2 or not all(isinstance(value, int) and not isinstance(value, bool) for value in self.job["rootIdentity"]):
raise AgentError("Invalid host data ownership proof")
if self.root.exists():
# The authorized device/inode is persisted before deletion so a
# crash halfway through rmtree does not require a deleted marker.
if not self.job.get("removalStarted"):
uninstall_ownership({"root": str(self.root), "controller": self.job["controller"], "hostId": host_id})
def save(self, **changes):
self.job.update(changes)
atomic_json(UNINSTALL_REQUEST, self.job)
def receipt(self, stage, error=None):
payload = {"stage": stage}
if error:
payload["error"] = error
request = urllib.request.Request(self.job["controller"].rstrip("/") + self.job["receiptUrl"], data=json.dumps(payload).encode(), headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.job["receiptToken"], "User-Agent": "electric-cherry-agent/" + VERSION}, method="POST")
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, hdrs, newurl):
raise AgentError("Controller redirects are not allowed")
try:
with urllib.request.build_opener(NoRedirect()).open(request, timeout=35) as response:
if json.load(response).get("acknowledged") is not True:
raise AgentError("Controller has not acknowledged host removal")
except (OSError, ValueError):
raise AgentError("Host removal receipt is waiting for controller contact") from None
def containers(self):
ids = self.docker.call("container", "ls", "--all", "--quiet", "--no-trunc").splitlines()
values = []
for container_id in ids:
info = self.docker.inspect(container_id)
if info:
values.append(info)
return values
def owned_container(self, info):
labels = info.get("Config", {}).get("Labels", {}) or {}
if labels.get(LABEL + ".managed") != "true":
return False
if labels.get(LABEL + ".host") not in (None, self.job["hostId"]):
return False
try:
instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", "")))
except (AgentError, ValueError, TypeError):
return False
expected = str(self.root / "instances" / instance_id / str(generation) / "app")
# Legacy agents did not label the host; its exact managed bind mount
# proves ownership without touching another installation's containers.
return any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", []))
def uses_root(self, info):
for mount in info.get("Mounts", []):
if mount.get("Type") != "bind":
continue
source = Path(mount.get("Source", "/")).resolve()
if source == self.root or self.root in source.parents:
return True
return False
def remove_docker_resources(self):
containers = self.containers()
if any(self.uses_root(info) and not self.owned_container(info) for info in containers):
raise AgentError("An unrelated container uses the host data directory; detach it before retrying deletion")
owned = [info for info in containers if self.owned_container(info)]
if any(info.get("State", {}).get("Running") is not False or info.get("State", {}).get("Status") not in ("created", "exited", "dead") for info in owned):
raise AgentError("A managed game container is still running or its state is unknown; retire it before deleting this host")
for info in owned:
# Do not force removal: Docker must also reject a container that
# starts again after inspection rather than interrupting players.
self.docker.call("container", "rm", "--volumes", info["Id"], timeout=120, tolerate_missing=True)
remaining = self.containers()
if any(self.uses_root(info) for info in remaining):
raise AgentError("Containers still use the host data directory")
# Never prune Docker globally. Only remove this agent's recorded image
# references, and preserve images used by any unrelated container.
used_images = {info.get("Image") for info in remaining}
images = self.job.get("runtimeImages", {})
if not isinstance(images, dict):
raise AgentError("Managed runtime image ownership is unavailable")
for reference, expected_id in images.items():
if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,511}", reference) or not isinstance(expected_id, str):
raise AgentError("Managed runtime image ownership is invalid")
raw = self.docker.call("image", "inspect", reference, tolerate_missing=True)
if raw is None:
continue
found = json.loads(raw)
if not found or found[0].get("Id") != expected_id or expected_id in used_images:
continue
self.docker.call("image", "rm", reference, tolerate_missing=True)
def remove_data(self):
protected_path(AGENT_CONFIG.parent, missing=True)
if not self.job.get("agentStopped"):
systemctl("disable", "--now", AGENT_UNIT.name)
self.save(agentStopped=True)
self.remove_docker_resources()
root = uninstall_root(str(self.root), self.job["rootIdentity"])
# This checkpoint also authorizes safe continuation if a power loss
# removes the ownership marker before the rest of the tree.
self.save(removalStarted=True)
if root.exists():
if not shutil.rmtree.avoids_symlink_attacks:
raise AgentError("This platform cannot safely remove managed host files")
shutil.rmtree(root)
sync_parent(root)
for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT):
protected_file(path, missing=True)
path.unlink(missing_ok=True)
sync_parent(path)
systemctl("daemon-reload")
self.save(dataRemoved=True)
def remove_helper(self):
# Disable does not stop this process: it must outlive its own unit and
# executable to deliver the final receipt after the files are gone.
protected_file(UNINSTALL_UNIT, missing=True)
if UNINSTALL_UNIT.exists():
systemctl("disable", UNINSTALL_UNIT.name)
UNINSTALL_UNIT.unlink()
sync_parent(UNINSTALL_UNIT)
systemctl("daemon-reload")
protected_file(UNINSTALL_EXECUTABLE, missing=True)
UNINSTALL_EXECUTABLE.unlink(missing_ok=True)
sync_parent(UNINSTALL_EXECUTABLE)
protected_path(UNINSTALL_REQUEST.parent, missing=True)
if UNINSTALL_REQUEST.parent.exists():
shutil.rmtree(UNINSTALL_REQUEST.parent)
sync_parent(UNINSTALL_REQUEST.parent)
def run(self):
if not self.job.get("dataRemoved"):
try:
self.remove_data()
except (AgentError, OSError, ValueError, KeyError) as error:
with contextlib.suppress(AgentError):
self.receipt("failed", str(error) if isinstance(error, AgentError) else "Host cleanup could not finish. Check the uninstall service and resolve filesystem errors; cleanup will retry.")
raise
self.receipt("cleaned")
while True:
try:
self.remove_helper()
break
except (AgentError, OSError, ValueError):
# Once its unit/executable has been unlinked systemd can no
# longer restart this worker. Finish idempotently in memory.
with contextlib.suppress(AgentError):
self.receipt("failed", "Host files were removed, but removing the uninstall service is still retrying")
time.sleep(15)
# The controller never marks a host deleted on the earlier receipt.
# No credentials or retry state are left on disk at this point.
while True:
try:
self.receipt("completed")
return
except AgentError:
time.sleep(15)
class Agent:
def __init__(self, config, docker=None):
self.config = config
@@ -300,6 +580,43 @@ class Agent:
self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
self.executing_id = None
def can_uninstall(self):
try:
if Path(self.config_path) != AGENT_CONFIG:
return False
uninstall_ownership(self.config)
protected_file(UNINSTALL_EXECUTABLE)
protected_file(UNINSTALL_UNIT)
return True
except (AgentError, OSError, ValueError):
return False
def decommission(self, operation):
payload = operation.get("payload", {})
if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]:
raise Rejected("Host removal identity does not match this agent")
if payload.get("receiptUrl") != "/api/agent/decommission/" + identifier(operation["id"]) or not isinstance(payload.get("receiptToken"), str) or not payload["receiptToken"]:
raise Rejected("Invalid host removal receipt")
if not self.can_uninstall():
raise Rejected("Reinstall the agent to enable safe removal of its dedicated host files")
root = uninstall_ownership(self.config)
info = root.stat()
protected_file(UNINSTALL_REQUEST, missing=True)
aliases_path = root / "agent" / "runtime-images.json"
protected_file(aliases_path, missing=True)
aliases = json.loads(aliases_path.read_text()) if aliases_path.exists() else {}
job = {"hostId": payload["hostId"], "operationId": operation["id"], "receiptUrl": payload["receiptUrl"], "receiptToken": payload["receiptToken"], "controller": self.config["controller"].rstrip("/"), "root": str(root), "rootIdentity": [info.st_dev, info.st_ino], "runtimeImages": aliases}
if UNINSTALL_REQUEST.exists():
previous = json.loads(UNINSTALL_REQUEST.read_text())
if any(previous.get(key) != job[key] for key in ("hostId", "operationId", "receiptUrl", "receiptToken", "controller", "root", "rootIdentity")):
raise Rejected("A different host removal is already pending")
self.checkpoint({"phase": "uninstall-service"})
if not UNINSTALL_REQUEST.exists():
atomic_json(UNINSTALL_REQUEST, job)
systemctl("enable", "--now", "--no-block", UNINSTALL_UNIT.name)
self.stopping.set()
return {"decommissioning": True, "message": "The uninstall service will report completion after host files and the agent are removed"}
def instance_dir(self, instance_id, generation):
instance_id, generation = instance_identity(instance_id, generation)
return self.root / "instances" / instance_id / str(generation)
@@ -416,7 +733,7 @@ class Agent:
for line in Path("/etc/os-release").read_text().splitlines():
if line.startswith("PRETTY_NAME="):
os_name = line.split("=", 1)[1].strip('"')
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
def safe_to_disrupt(self, instance_id, generation):
health = self.inspect(instance_id, generation)
@@ -525,6 +842,8 @@ class Agent:
name = self.container_name(instance_id, generation)
spec_sha = digest(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode())
args = ["container", "create", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)]
if self.config.get("hostId"):
args.extend(["--label", LABEL + ".host=" + identifier(self.config["hostId"])])
bind_arg = "[" + bind + "]" if ":" in bind else bind
for protocol in ("udp", "tcp"):
args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
@@ -923,6 +1242,8 @@ class Agent:
kind, payload = operation.get("kind"), operation.get("payload", {})
if not isinstance(kind, str) or not isinstance(payload, dict):
raise AgentError("Invalid operation")
if kind == "host.decommission":
return self.decommission(operation)
if kind == "credentials.rotate":
pending = "ec_host_" + base64.urlsafe_b64encode(os.urandom(32)).rstrip(b"=").decode()
self.config["pendingCredential"] = pending
@@ -997,7 +1318,7 @@ class Agent:
return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message)
finally:
self.executing_id = None
return self.journal.update(operation["id"], "completed", result)
return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result)
def recover_credential(self):
pending = self.config.get("pendingCredential")
@@ -1024,6 +1345,13 @@ class Agent:
instance_id, generation = operation.get("instanceId"), operation.get("generation")
checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
try:
if kind == "host.decommission":
self.executing_id = operation["id"]
try:
result = self.decommission(operation)
return self.journal.update(operation["id"], "running", result)
finally:
self.executing_id = None
if kind == "credentials.rotate":
if not self.config.get("pendingCredential") or self.recover_credential():
return self.journal.update(operation["id"], "completed", {"rotated": True, "reconciled": True})
@@ -1183,8 +1511,14 @@ def main():
parser.add_argument("--token-file")
parser.add_argument("--root", default="/var/lib/electric-cherry")
parser.add_argument("--force-enroll", action="store_true")
parser.add_argument("--uninstall", metavar="REQUEST", help="Run the installed host removal service")
parser.add_argument("--version", action="version", version=VERSION)
args = parser.parse_args()
if args.uninstall:
if os.geteuid() != 0 or args.enroll:
raise AgentError("Host uninstall must run as the dedicated root service")
Uninstaller(args.uninstall).run()
return
if args.enroll:
if not args.controller or not args.token_file:
parser.error("Enrollment needs --controller and --token-file")
+120 -7
View File
@@ -21,7 +21,7 @@ done
case "$controller" in https://*) ;; *) echo '--controller must be an HTTPS URL.' >&2; exit 2 ;; esac
[ -n "$enrollment_token" ] || { echo '--token is required.' >&2; exit 2; }
case "$data_dir" in /*) ;; *) echo '--data-dir must be absolute.' >&2; exit 2 ;; esac
case "$data_dir" in *[!a-zA-Z0-9/_.-]*|/|/etc|/usr|/var|/var/lib) echo 'Choose a dedicated data directory.' >&2; exit 2 ;; esac
case "$data_dir" in *[!a-zA-Z0-9/_.-]*|*/../*|*/./*|*/..|*/.|*//*|*/) echo 'Choose a canonical, dedicated data directory.' >&2; exit 2 ;; esac
[ ! -L "$data_dir" ] || { echo 'Data directory must not be a symlink.' >&2; exit 2; }
. /etc/os-release
[ "${ID:-}" = debian ] || { echo 'Supported platform: Debian 12 or 13.' >&2; exit 1; }
@@ -32,13 +32,82 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; }
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y --no-install-recommends ca-certificates curl python3
python3 - "$controller" "$data_dir" <<'PY'
import json, pathlib, sys
config = pathlib.Path('/etc/electric-cherry/agent.json')
if config.exists():
saved = json.loads(config.read_text())
if saved.get('controller', '').rstrip('/') != sys.argv[1].rstrip('/') or pathlib.Path(saved.get('root', '')).resolve() != pathlib.Path(sys.argv[2]).resolve():
python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY'
# INSTALLATION_PREFLIGHT: this runs before touching managed directories or services.
import json, pathlib, re, stat, sys
controller, raw_root, config_name = sys.argv[1:]
root = pathlib.Path(raw_root)
config = pathlib.Path(config_name)
reserved = {
'/', '/bin', '/boot', '/dev', '/etc', '/home', '/lib', '/lib64',
'/media', '/mnt', '/opt', '/proc', '/root', '/run', '/sbin', '/srv',
'/sys', '/tmp', '/usr', '/usr/local', '/usr/local/bin', '/usr/local/lib',
'/usr/local/libexec', '/var', '/var/cache', '/var/lib', '/var/log', '/var/tmp',
}
if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root)
or str(root) != raw_root or any(part in ('.', '..') for part in raw_root.split('/')[1:])
or len(root.parts) < 3 or raw_root in reserved or config.parent == root or config.parent in root.parents):
raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.')
installed_paths = [config,
pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'),
pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'),
pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'),
pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service')]
if any(root == path or root in path.parents for path in installed_paths):
raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.')
def trusted_directories(path):
for current in [*reversed(path.parents), path]:
try:
info = current.lstat()
except FileNotFoundError:
continue
if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.')
def trusted_json(path):
info = path.lstat()
if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.')
value = json.loads(path.read_text())
if not isinstance(value, dict):
raise SystemExit('Existing installation metadata is invalid.')
return value
trusted_directories(root)
trusted_directories(config.parent)
mountinfo = pathlib.Path('/proc/1/mountinfo')
if mountinfo.exists():
for line in mountinfo.read_text().splitlines():
fields = line.split()
if len(fields) < 5:
continue
mount = re.sub(r'\\([0-7]{3})', lambda match: chr(int(match[1], 8)), fields[4])
if mount == raw_root or mount.startswith(raw_root + '/'):
raise SystemExit('Use a dedicated subdirectory of the mounted disk, with no filesystems mounted inside it, so host deletion can remove the entire data directory.')
request = config.with_name('decommission.json')
if request.exists() or request.is_symlink():
raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.')
saved = None
if config.exists() or config.is_symlink():
saved = trusted_json(config)
if saved.get('controller', '').rstrip('/') != controller.rstrip('/') or saved.get('root') != raw_root:
raise SystemExit('Existing enrollment uses a different controller or data directory. Preserve the existing installation and migrate it explicitly.')
if not saved.get('hostId') or not saved.get('credential'):
raise SystemExit('Existing enrollment is incomplete. Repair its credentials before upgrading.')
marker = root / '.electric-cherry.json'
if marker.exists() or marker.is_symlink():
owned = trusted_json(marker)
if not saved or owned != {'version': 1, 'root': raw_root, 'hostId': saved['hostId'], 'controller': controller.rstrip('/')}:
raise SystemExit('Data directory ownership does not match this enrollment. Preserve it and migrate explicitly.')
elif root.exists() and any(root.iterdir()):
# Upgrade installations predating ownership markers only when the entire
# root has the known layout. Never adopt a directory of unrelated files.
if not saved or any(path.name not in {'agent', 'instances', 'libraries'} for path in root.iterdir()):
raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.')
for directory in ('agent', 'instances', 'libraries'):
trusted_directories(root / directory)
PY
if ! command -v docker >/dev/null; then
if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then
@@ -76,9 +145,36 @@ install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-
install -d -m 0755 /usr/local/libexec
install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py
}
# A separate worker survives stopping/removing the agent service. Its only
# entry point is a fixed, authenticated deletion request prepared by the agent.
install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py
printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token"
unset enrollment_token
python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"}
python3 - "$data_dir" /etc/electric-cherry/agent.json <<'PY'
# INSTALLATION_OWNERSHIP: bind deletion authority to this exact enrollment/root.
import json, os, pathlib, sys, uuid
root = pathlib.Path(sys.argv[1])
config = json.loads(pathlib.Path(sys.argv[2]).read_text())
if config.get('root') != str(root) or not config.get('hostId') or not config.get('controller'):
raise SystemExit('Enrollment did not establish ownership of this data directory.')
marker = root / '.electric-cherry.json'
temporary = root / ('.electric-cherry.' + uuid.uuid4().hex + '.tmp')
try:
with temporary.open('x') as stream:
os.chmod(temporary, 0o600)
json.dump({'version': 1, 'root': str(root), 'hostId': config['hostId'], 'controller': config['controller'].rstrip('/')}, stream)
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, marker)
directory_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
finally:
temporary.unlink(missing_ok=True)
PY
cat > /etc/systemd/system/electric-cherry-agent.service <<EOF
[Unit]
Description=Electric Cherry Host Agent
@@ -99,6 +195,23 @@ ProtectHome=true
PrivateTmp=true
ReadWritePaths=$data_dir /etc/electric-cherry
[Install]
WantedBy=multi-user.target
EOF
cat > /etc/systemd/system/electric-cherry-uninstall.service <<'EOF'
[Unit]
Description=Remove an Electric Cherry Host Installation
After=network-online.target docker.service
Wants=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-uninstall.py --uninstall /etc/electric-cherry/decommission.json
Restart=on-failure
RestartSec=15
TimeoutStartSec=0
UMask=0077
[Install]
WantedBy=multi-user.target
EOF
+1 -1
View File
@@ -10,4 +10,4 @@ target = source.parent / "static" / "agent"
target.mkdir(parents=True, exist_ok=True)
shutil.copyfile(source / "ec_agent.py", target / "ec-agent.py")
shutil.copyfile(source / "install.sh", target / "install.sh")
(target / "manifest.json").write_text(json.dumps({"version": "1.0.0", "protocolVersion": 1, "platform": "debian-amd64", "sha256": hashlib.sha256((target / "ec-agent.py").read_bytes()).hexdigest()}, indent=2) + "\n")
(target / "manifest.json").write_text(json.dumps({"version": "1.1.0", "protocolVersion": 1, "platform": "debian-amd64", "sha256": hashlib.sha256((target / "ec-agent.py").read_bytes()).hexdigest()}, indent=2) + "\n")
+185
View File
@@ -0,0 +1,185 @@
"""Exercise installer guards without installing software or touching services."""
import json
import os
from pathlib import Path
import re
import stat
import sys
import tempfile
import unittest
from unittest.mock import patch
SOURCE = (Path(__file__).resolve().parents[1] / "install.sh").read_text()
BLOCKS = re.findall(r"<<'PY'\n(.*?)\nPY", SOURCE, re.DOTALL)
PREFLIGHT = next(block for block in BLOCKS if "# INSTALLATION_PREFLIGHT:" in block)
OWNERSHIP = next(block for block in BLOCKS if "# INSTALLATION_OWNERSHIP:" in block)
class InstallerTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.base = Path(self.temporary.name).resolve()
self.root = self.base / "managed"
self.config = self.base / "configuration" / "agent.json"
self.config.parent.mkdir()
self.controller = "https://controller.example"
self.saved = {"root": str(self.root), "controller": self.controller,
"hostId": "host-one", "credential": "private"}
self.overrides = {}
def tearDown(self):
self.temporary.cleanup()
def preflight(self, root=None):
original_lstat = Path.lstat
def root_owned(path):
# Simulate Linux root-owned parents while keeping the actual inode
# types. No filesystem mutations or elevated privileges are needed.
info = original_lstat(path)
fields = list(info)
fields[4] = 0
if stat.S_ISDIR(info.st_mode):
fields[0] &= ~0o022
override = self.overrides.get(str(path), {})
fields[0] = override.get("mode", fields[0])
fields[4] = override.get("uid", fields[4])
return os.stat_result(fields)
argv = ["-", self.controller, str(root or self.root), str(self.config)]
with patch.object(sys, "argv", argv), patch.object(Path, "lstat", root_owned):
exec(compile(PREFLIGHT, "installer-preflight", "exec"), {})
def legacy_installation(self):
for directory in ("agent", "instances", "libraries"):
(self.root / directory).mkdir(parents=True, exist_ok=True)
self.config.write_text(json.dumps(self.saved))
def ownership(self):
with patch.object(sys, "argv", ["-", str(self.root), str(self.config)]):
exec(compile(OWNERSHIP, "installer-ownership", "exec"), {})
def test_new_directory_and_empty_directory_are_allowed_without_adopting_files(self):
self.preflight()
self.assertFalse(self.root.exists())
self.root.mkdir()
self.preflight()
(self.root / "unrelated.txt").write_text("keep")
with self.assertRaisesRegex(SystemExit, "not owned"):
self.preflight()
self.assertEqual((self.root / "unrelated.txt").read_text(), "keep")
def test_known_legacy_layout_can_upgrade_and_marker_binds_exact_enrollment(self):
self.legacy_installation()
original_config = self.config.read_bytes()
(self.root / "libraries" / "game.ff").write_bytes(b"game data")
self.preflight()
self.ownership()
marker = self.root / ".electric-cherry.json"
self.assertEqual(json.loads(marker.read_text()), {
"version": 1, "root": str(self.root), "hostId": "host-one", "controller": self.controller})
self.assertEqual(stat.S_IMODE(marker.stat().st_mode), 0o600)
self.assertEqual(self.config.read_bytes(), original_config)
self.assertEqual((self.root / "libraries" / "game.ff").read_bytes(), b"game data")
self.preflight()
def test_legacy_upgrade_never_adopts_unrelated_top_level_files(self):
self.legacy_installation()
(self.root / "personal-backup.tar").write_bytes(b"keep")
with self.assertRaisesRegex(SystemExit, "not owned"):
self.preflight()
self.assertFalse((self.root / ".electric-cherry.json").exists())
def test_reenrollment_cannot_silently_change_controller_or_root(self):
self.legacy_installation()
for field, value in (("controller", "https://another.example"), ("root", str(self.base / "other"))):
with self.subTest(field=field):
self.config.write_text(json.dumps({**self.saved, field: value}))
with self.assertRaisesRegex(SystemExit, "different controller or data directory"):
self.preflight()
def test_existing_marker_must_match_controller_host_and_root(self):
self.legacy_installation()
self.ownership()
marker = self.root / ".electric-cherry.json"
valid = json.loads(marker.read_text())
for field, value in (("hostId", "host-two"), ("root", str(self.base)), ("controller", "https://another.example")):
with self.subTest(field=field):
marker.write_text(json.dumps({**valid, field: value}))
with self.assertRaisesRegex(SystemExit, "ownership does not match"):
self.preflight()
def test_lexical_aliases_and_system_roots_are_rejected_before_mutation(self):
for root in ("/var/lib/..", "/var/lib/../lib", "/var//lib/managed", "/var/lib/./managed", "/var/lib/managed/", "/var/lib", "/usr/local/libexec", "/", "/games", str(self.config.parent), str(self.config.parent / "data")):
with self.subTest(root=root), self.assertRaisesRegex(SystemExit, "canonical, dedicated"):
self.preflight(root)
self.assertFalse(self.root.exists())
def test_symlink_roots_ancestors_metadata_and_managed_subdirectories_are_rejected(self):
elsewhere = self.base / "elsewhere"
elsewhere.mkdir()
self.root.symlink_to(elsewhere, target_is_directory=True)
with self.assertRaisesRegex(SystemExit, "not symlinks"):
self.preflight()
with self.assertRaisesRegex(SystemExit, "not symlinks"):
self.preflight(str(self.root / "nested"))
self.root.unlink()
self.legacy_installation()
(self.root / "libraries").rmdir()
(self.root / "libraries").symlink_to(elsewhere, target_is_directory=True)
with self.assertRaisesRegex(SystemExit, "not symlinks"):
self.preflight()
(self.root / "libraries").unlink()
(self.root / "libraries").mkdir()
self.config.unlink()
real_config = self.base / "real-config.json"
real_config.write_text(json.dumps(self.saved))
self.config.symlink_to(real_config)
with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
self.preflight()
def test_untrusted_parents_and_metadata_cannot_authorize_cleanup(self):
self.legacy_installation()
self.overrides[str(self.base)] = {"uid": 1000}
with self.assertRaisesRegex(SystemExit, "root-owned"):
self.preflight()
self.overrides = {str(self.base): {"mode": stat.S_IFDIR | 0o777}}
with self.assertRaisesRegex(SystemExit, "root-owned"):
self.preflight()
self.overrides = {str(self.config): {"uid": 1000}}
with self.assertRaisesRegex(SystemExit, "regular, root-owned"):
self.preflight()
def test_pending_deletion_blocks_upgrade_even_with_dangling_request_symlink(self):
self.legacy_installation()
request = self.config.with_name("decommission.json")
request.write_text("{}")
with self.assertRaisesRegex(SystemExit, "deletion is in progress"):
self.preflight()
request.unlink()
request.symlink_to(self.base / "missing")
with self.assertRaisesRegex(SystemExit, "deletion is in progress"):
self.preflight()
def test_mounted_parent_is_allowed_but_data_root_and_nested_mounts_are_rejected(self):
mountinfo = Path('/proc/1/mountinfo')
original_exists, original_read = Path.exists, Path.read_text
for mount, allowed in ((self.base, True), (self.root, False), (self.root / 'libraries', False)):
def exists(path):
return True if path == mountinfo else original_exists(path)
def read(path, *args, **kwargs):
return f'29 24 8:1 / {mount} rw - ext4 /dev/sda1 rw' if path == mountinfo else original_read(path, *args, **kwargs)
with self.subTest(mount=mount), patch.object(Path, 'exists', exists), patch.object(Path, 'read_text', read):
if allowed:
self.preflight()
else:
with self.assertRaisesRegex(SystemExit, "dedicated subdirectory of the mounted disk"):
self.preflight()
if __name__ == "__main__":
unittest.main()
+338
View File
@@ -0,0 +1,338 @@
import importlib.util
import json
import os
from pathlib import Path
import shutil
import stat
import tempfile
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location("ec_uninstall_agent", Path(__file__).resolve().parents[1] / "ec_agent.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
INSTANCE = "a53d99da-a1eb-4f01-9b10-a6268b556542"
class CleanupDocker:
def __init__(self):
self.calls = []
self.items = {}
self.images = {}
def inspect(self, identity):
return self.items.get(identity)
def call(self, *args, **kwargs):
self.calls.append(args)
if args[:2] == ("container", "ls"):
return "\n".join(self.items)
if args[:2] == ("container", "rm"):
self.items.pop(args[-1], None)
return ""
if args[:2] == ("image", "inspect"):
item = self.images.get(args[-1])
return json.dumps([item]) if item else None
if args[:2] == ("image", "rm"):
self.images.pop(args[-1], None)
return ""
raise AssertionError("Unexpected Docker request: " + str(args))
class UninstallTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.base = Path(self.temp.name).resolve()
self.root = self.base / "host-data"
self.config_dir = self.base / "etc" / "electric-cherry"
self.bin_dir = self.base / "libexec"
self.unit_dir = self.base / "systemd"
for directory in (self.config_dir, self.bin_dir, self.unit_dir):
directory.mkdir(parents=True)
replacements = {
"AGENT_CONFIG": self.config_dir / "agent.json",
"AGENT_EXECUTABLE": self.bin_dir / "electric-cherry-agent.py",
"AGENT_UNIT": self.unit_dir / "electric-cherry-agent.service",
"UNINSTALL_EXECUTABLE": self.bin_dir / "electric-cherry-uninstall.py",
"UNINSTALL_UNIT": self.unit_dir / "electric-cherry-uninstall.service",
"UNINSTALL_REQUEST": self.config_dir / "decommission.json",
}
self.patches = [patch.object(module, key, value) for key, value in replacements.items()]
# Unit tests run without root privileges. Keep actual path, symlink,
# file-type and containment checks while substituting owner checks.
def test_protected_path(value, missing=False):
path = Path(value)
if any(current.is_symlink() for current in (path, *path.parents)):
raise module.AgentError("Symlink ownership is invalid")
if not missing and not path.exists():
raise module.AgentError("Host uninstall files are unavailable")
return path
self.patches.extend([patch.object(module, "protected_path", side_effect=test_protected_path), patch.object(module, "systemctl"), patch.object(module, "sync_parent")])
for entry in self.patches:
entry.start()
self.config = {"hostId": "host-one", "controller": "https://controller.example", "credential": "secret", "root": str(self.root), "configPath": str(module.AGENT_CONFIG)}
self.agent = module.Agent(self.config)
self.docker = CleanupDocker()
for path in (module.AGENT_EXECUTABLE, module.AGENT_UNIT, module.UNINSTALL_EXECUTABLE, module.UNINSTALL_UNIT):
path.write_text("installed")
module.atomic_json(module.AGENT_CONFIG, self.config)
module.atomic_json(self.root / module.ROOT_MARKER, {"version": 1, "root": str(self.root), "hostId": "host-one", "controller": "https://controller.example"})
self.game = self.root / "libraries" / "t6" / "library" / "content" / "zone" / "all" / "dlc.ff"
self.game.parent.mkdir(parents=True)
self.game.write_bytes(b"game DLC")
(self.base / "unrelated.txt").write_text("keep")
self.operation = {"id": "delete-one", "kind": "host.decommission", "payload": {"hostId": "host-one", "operationId": "delete-one", "receiptToken": "receipt-only-secret", "receiptUrl": "/api/agent/decommission/delete-one"}}
def tearDown(self):
self.agent.journal.close()
for entry in reversed(self.patches):
entry.stop()
self.temp.cleanup()
def prepare(self):
result = self.agent.execute(self.operation)
self.assertEqual(result["state"], "running", result)
return module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
def container(self, identity, owned=True, host="host-one", root=None, image="image-one"):
labels = {module.LABEL + ".managed": "true", module.LABEL + ".instance": INSTANCE, module.LABEL + ".generation": "1", module.LABEL + ".host": host} if owned else {}
return {"Id": identity, "Image": image, "Config": {"Labels": labels}, "State": {"Running": False, "Status": "exited"}, "Mounts": [{"Type": "bind", "Source": str((root or self.root) / "instances" / INSTANCE / "1" / "app"), "Destination": "/home/plutainer/app"}]}
def test_final_receipt_follows_data_and_self_removal(self):
worker = self.prepare()
stages = []
def receipt(stage, error=None):
stages.append(stage)
self.assertFalse(self.root.exists())
self.assertFalse(module.AGENT_CONFIG.exists())
self.assertFalse(module.AGENT_EXECUTABLE.exists())
self.assertFalse(module.AGENT_UNIT.exists())
if stage == "cleaned":
self.assertTrue(module.UNINSTALL_REQUEST.exists())
self.assertTrue(module.UNINSTALL_EXECUTABLE.exists())
else:
self.assertFalse(self.config_dir.exists())
self.assertFalse(module.UNINSTALL_EXECUTABLE.exists())
self.assertFalse(module.UNINSTALL_UNIT.exists())
worker.receipt = receipt
worker.run()
self.assertEqual(stages, ["cleaned", "completed"])
self.assertEqual((self.base / "unrelated.txt").read_text(), "keep")
module.systemctl.assert_any_call("disable", "--now", "electric-cherry-agent.service")
module.systemctl.assert_any_call("disable", "electric-cherry-uninstall.service")
def test_handoff_is_durable_and_never_reports_operation_completed(self):
worker = self.prepare()
self.assertEqual(worker.job["operationId"], "delete-one")
self.assertNotIn("credential", worker.job)
self.assertTrue(self.agent.stopping.is_set())
module.systemctl.assert_any_call("enable", "--now", "--no-block", "electric-cherry-uninstall.service")
self.assertEqual(self.agent.journal.get("delete-one")["state"], "running")
def test_failed_checkpoint_cannot_leave_a_runnable_uninstall_request(self):
with patch.object(self.agent.journal, "checkpoint", side_effect=OSError("journal unavailable")):
result = self.agent.execute(self.operation)
self.assertEqual(result["state"], "failed")
self.assertFalse(module.UNINSTALL_REQUEST.exists())
module.systemctl.assert_not_called()
def test_pending_cleanup_resumes_after_controller_unavailable(self):
worker = self.prepare()
worker.receipt = lambda *_: (_ for _ in ()).throw(module.AgentError("offline"))
with self.assertRaisesRegex(module.AgentError, "offline"):
worker.run()
self.assertFalse(self.root.exists())
self.assertTrue(module.UNINSTALL_REQUEST.exists())
self.assertTrue(module.UNINSTALL_EXECUTABLE.exists())
self.assertTrue(json.loads(module.UNINSTALL_REQUEST.read_text())["dataRemoved"])
before = len(self.docker.calls)
resumed = module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
stages = []
resumed.receipt = lambda stage, error=None: stages.append(stage)
resumed.run()
self.assertEqual(stages, ["cleaned", "completed"])
self.assertEqual(len(self.docker.calls), before)
def test_lost_final_receipt_retries_after_all_files_are_gone(self):
worker = self.prepare()
stages = []
def receipt(stage, error=None):
stages.append(stage)
if stage == "completed":
self.assertFalse(self.config_dir.exists())
if stages.count("completed") == 1:
raise module.AgentError("lost reply")
worker.receipt = receipt
with patch.object(module.time, "sleep") as wait:
worker.run()
self.assertEqual(stages, ["cleaned", "completed", "completed"])
wait.assert_called_once_with(15)
def test_partial_self_removal_retries_without_its_unit_or_executable(self):
worker = self.prepare()
stages = []
worker.receipt = lambda stage, error=None: stages.append(stage)
original = shutil.rmtree
attempts = 0
def interrupted(path, *args, **kwargs):
nonlocal attempts
if Path(path) == self.config_dir:
attempts += 1
if attempts == 1:
self.assertFalse(module.UNINSTALL_UNIT.exists())
self.assertFalse(module.UNINSTALL_EXECUTABLE.exists())
raise OSError("temporarily busy")
return original(path, *args, **kwargs)
with patch.object(module.shutil, "rmtree", side_effect=interrupted), patch.object(module.time, "sleep") as wait:
worker.run()
self.assertEqual(stages, ["cleaned", "failed", "completed"])
self.assertFalse(self.config_dir.exists())
wait.assert_called_once_with(15)
def test_partial_tree_removal_recovers_without_deleted_marker(self):
worker = self.prepare()
original = shutil.rmtree
def interrupted(path, *args, **kwargs):
if Path(path) == self.root:
(self.root / module.ROOT_MARKER).unlink()
self.game.unlink()
raise OSError("power loss")
return original(path, *args, **kwargs)
worker.receipt = lambda *_: None
with patch.object(module.shutil, "rmtree", side_effect=interrupted):
with self.assertRaisesRegex(OSError, "power loss"):
worker.run()
self.assertTrue(json.loads(module.UNINSTALL_REQUEST.read_text())["removalStarted"])
self.assertFalse((self.root / module.ROOT_MARKER).exists())
resumed = module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
resumed.receipt = lambda *_: None
resumed.run()
self.assertFalse(self.root.exists())
def test_replaced_data_root_cannot_be_deleted_on_resume(self):
self.prepare()
self.root.rename(self.base / "original-data")
self.root.mkdir()
(self.root / "precious.txt").write_text("keep")
with self.assertRaisesRegex(module.AgentError, "changed"):
module.Uninstaller(module.UNINSTALL_REQUEST, self.docker)
self.assertTrue((self.root / "precious.txt").exists())
def test_only_owned_containers_and_unused_recorded_images_are_removed(self):
aliases = {"example/runtime:one": "image-one", "example/runtime:shared": "image-shared", "example/runtime:changed": "image-old"}
module.atomic_json(self.root / "agent" / "runtime-images.json", aliases)
self.docker.images = {"example/runtime:one": {"Id": "image-one"}, "example/runtime:shared": {"Id": "image-shared"}, "example/runtime:changed": {"Id": "image-new"}, "example/unrelated:one": {"Id": "image-unrelated"}}
self.docker.items = {"ours": self.container("ours"), "other": self.container("other", host="host-two", root=self.base / "different-install", image="image-shared")}
worker = self.prepare()
worker.receipt = lambda *_: None
worker.run()
self.assertNotIn("ours", self.docker.items)
self.assertIn("other", self.docker.items)
self.assertNotIn("example/runtime:one", self.docker.images)
self.assertIn("example/runtime:shared", self.docker.images)
self.assertIn("example/runtime:changed", self.docker.images)
self.assertIn("example/unrelated:one", self.docker.images)
self.assertFalse(any("prune" in call or "volume" in call for call in self.docker.calls))
def test_unrelated_container_using_root_blocks_any_removal(self):
self.docker.items = {"foreign": self.container("foreign", owned=False)}
worker = self.prepare()
stages = []
worker.receipt = lambda stage, error=None: stages.append((stage, error))
with self.assertRaisesRegex(module.AgentError, "unrelated container"):
worker.run()
self.assertEqual(stages[0][0], "failed")
self.assertTrue(self.game.exists())
self.assertTrue(module.AGENT_EXECUTABLE.exists())
self.assertIn("foreign", self.docker.items)
def test_running_or_unknown_managed_container_blocks_removal(self):
worker = self.prepare()
worker.receipt = lambda *_: None
for state in ({"Running": True, "Status": "running"}, {"Running": False, "Status": "restarting"}, {}):
with self.subTest(state=state):
self.docker.items = {"ours": {**self.container("ours"), "State": state}}
with self.assertRaisesRegex(module.AgentError, "still running or its state is unknown"):
worker.run()
self.assertTrue(self.game.exists())
self.assertIn("ours", self.docker.items)
self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
def test_legacy_container_requires_exact_managed_bind_mount(self):
worker = self.prepare()
legacy = self.container("legacy")
legacy["Config"]["Labels"].pop(module.LABEL + ".host")
self.assertTrue(worker.owned_container(legacy))
legacy["Mounts"][0]["Source"] = str(self.base / "elsewhere")
self.assertFalse(worker.owned_container(legacy))
def test_missing_marker_and_wrong_host_never_schedule_helper(self):
(self.root / module.ROOT_MARKER).unlink()
self.assertFalse(self.agent.can_uninstall())
result = self.agent.execute(self.operation)
self.assertEqual(result["state"], "failed")
module.systemctl.assert_not_called()
self.assertTrue(self.game.exists())
def test_receipt_path_and_identity_cannot_be_redirected(self):
for key, value in (("receiptUrl", "https://attacker.example"), ("hostId", "other"), ("operationId", "other")):
with self.subTest(key=key):
operation = {**self.operation, "id": "check-" + key, "payload": {**self.operation["payload"], "operationId": "check-" + key, "receiptUrl": "/api/agent/decommission/check-" + key, key: value}}
self.assertEqual(self.agent.execute(operation)["state"], "failed")
module.systemctl.assert_not_called()
def test_symlink_inside_owned_tree_is_unlinked_without_following(self):
outside = self.base / "unrelated.txt"
(self.root / "libraries" / "symlink").symlink_to(outside)
worker = self.prepare()
worker.receipt = lambda *_: None
worker.run()
self.assertEqual(outside.read_text(), "keep")
def test_unsafe_roots_and_installation_overlap_are_rejected(self):
for value in ("/", "/var/lib", "/srv", "/var/lib/../lib", "/var//lib/anything", str(self.config_dir), str(self.bin_dir)):
with self.subTest(value=value):
with self.assertRaises(module.AgentError):
module.uninstall_root(value)
def test_root_mount_or_nested_mount_cannot_be_recursively_deleted(self):
original = Path.read_text
for mount in (self.root, self.root / "libraries"):
def read_text(path, *args, **kwargs):
if str(path) == "/proc/1/mountinfo":
return "21 1 8:1 / " + str(mount) + " rw - ext4 /dev/sda1 rw\n"
return original(path, *args, **kwargs)
with patch.object(Path, "read_text", read_text):
with self.assertRaisesRegex(module.AgentError, "Unmount"):
module.uninstall_root(str(self.root))
def test_marker_directory_is_rejected_before_reading(self):
marker = self.root / module.ROOT_MARKER
marker.unlink()
marker.mkdir()
with self.assertRaisesRegex(module.AgentError, "regular file"):
module.uninstall_ownership(self.config)
def test_boot_reconciles_handoff_instead_of_marking_complete(self):
self.prepare()
self.agent.journal.close()
self.agent = module.Agent(self.config)
self.assertEqual(self.agent.journal.get("delete-one")["state"], "unknown")
result = self.agent.execute(self.operation)
self.assertEqual(result["state"], "running")
self.assertEqual(json.loads(module.UNINSTALL_REQUEST.read_text())["operationId"], "delete-one")
class OwnershipPathTests(unittest.TestCase):
def test_root_ownership_group_writes_and_symlinks_are_rejected(self):
for mode, uid in ((stat.S_IFDIR | 0o777, 0), (stat.S_IFDIR | 0o755, 1000), (stat.S_IFLNK | 0o777, 0)):
with self.subTest(mode=mode, uid=uid):
value = os.stat_result((mode, 1, 1, 1, uid, 0, 0, 0, 0, 0))
with patch.object(Path, "lstat", return_value=value):
with self.assertRaisesRegex(module.AgentError, "only by root"):
module.protected_path("/var/lib/electric-cherry")
if __name__ == "__main__":
unittest.main()
+4 -1
View File
@@ -31,7 +31,7 @@ guessable token. An unset setup token holds setup closed.
| Raw configs, file diffs, logs and telemetry diagnostics | No | Yes | Yes |
| Deploy/start/stop/restart/rebuild/remove servers | No | Yes | Yes |
| Scripts, plugins, configs, campaigns, betas and collections | Read summaries | Manage | Manage |
| Host setup, enrollment, credentials, runtime preparation and game-file import/cache | Read status | Read status | Manage |
| Host setup, deletion, enrollment, credentials, runtime preparation and game-file upload/cache | Read status | Read status | Manage |
| Server keys and global settings/maintenance policies | No | No | Manage |
| Create users, change roles, disable accounts and reset passwords | No | No | Yes |
| Own password and sign out | Yes | Yes | Yes |
@@ -85,6 +85,9 @@ endpoint or accept passwords as command arguments.
Host agents retain enrollment/host-token authentication under `/api/agent/`.
Installer and release files remain public so a new machine can enroll. A browser
login never substitutes for a host credential, and host tokens cannot manage EC.
Host deletion uses a separate operation-scoped receipt token so the uninstall
worker can confirm removal and retry its acknowledgment after host credentials
are revoked. That token authorizes only the corresponding cleanup receipt.
Game reporters must send a separate secret of at least 32 characters configured
as `EC_TELEMETRY_TOKEN`:
+1 -1
View File
@@ -131,7 +131,7 @@ the typed key so it cannot accidentally be saved against a different lobby.
Komodo connection settings, target host, RCON configuration, config filename,
lobby availability, and port availability are checked locally. These checks do
not probe the host or verify game/DLC files, images, Docker volumes, firewall
not probe the host or verify game files, images, Docker volumes, firewall
rules, or credentials. Container and RCON verification happen after provisioning.
T6 variants retain their exact catalog slug and launch ruleset; RCON health checks
verify the engine map, not the variant's start location.
+92 -11
View File
@@ -17,12 +17,16 @@ the agent owns local execution and its durable operation journal.
3. The installer validates the platform, installs Python 3 and an absent Docker
installation, verifies the downloaded agent SHA-256 over HTTPS, enrolls it and
enables `electric-cherry-agent.service`. It needs no Git, Node.js or local build.
4. In **Hosts**, prepare a Plutainer runtime image. In **Game Files**, import your
existing `t5_game_files` or `t6_game_files` Docker volume on an enrolled source
host. Name the library and version and record its DLC metadata.
4. In **Hosts**, prepare a Plutainer runtime image. In **Game Files**, select T5 or
T6 and upload your local game folder. Name the library and version and publish
it after the upload finishes. Include the DLC files in that folder; there is
no separate DLC declaration. Uploading also works before enrolling any host.
5. Cache the published version on each destination host. Wait for verification.
Hosts report connection and readiness separately; deployment also checks the
particular library, image, capacity and ports you selected.
particular library, image, capacity and ports you selected. A connected host
missing content shows **Needs runtime** or **Needs game files**; these require
setup rather than waiting for an automatic preparation step. **Ready** means
it has both a runtime image and a verified library available.
6. Launch a server or collection. Select the host, immutable game-file version(s)
and runtime image. EC selects lobby IDs and reserves UDP/TCP ports atomically.
Saved Plutonium server keys remain associated with their exact server names.
@@ -31,7 +35,10 @@ A fresh destination host downloads centrally stored game files without manual
copying or per-container setup. EC does not supply the game content itself.
The default host root is `/var/lib/electric-cherry`; installer `--data-dir` changes
it. Data is separated into `libraries/<game>/<version>/content`,
it. Use a dedicated, root-owned directory. For a separate disk, use a child of its
mount point (for example `/mnt/games/electric-cherry`), with no nested mounts, so
host deletion can remove the entire directory. The installer records ownership
and refuses to adopt unrelated files. Data is separated into `libraries/<game>/<version>/content`,
`instances/<UUID>/<generation>/app`, and `agent/operations.sqlite3`. Base libraries
are mounted read-only and instance deletion removes only that instance's files.
The controller stores large files under `GAME_FILES_ROOT` (default beside the
@@ -69,12 +76,25 @@ explicit installer `--reenroll` when local credentials already exist.
API outline:
- `GET /api/hosts/:id`: host details, operation history and deletion blockers.
- `DELETE /api/hosts/:id`: administrator requests cleanup of an unused host;
returns `202` while remote removal is pending, `200` for an unenrolled host,
or `409` when the agent needs updating or unresolved ownership remains.
- `POST /api/agent/enroll`: consume enrollment token and return host credentials.
- `POST /api/agent/heartbeat`: authenticated inventory, current operation and caches.
- `GET /api/agent/operations`: deliver/reconcile durable operation IDs.
- `POST /api/agent/operations/:id`: authenticated acceptance, progress or result.
- `/api/agent/imports/...` and `/api/agent/libraries/...`: host-scoped uploads and
authenticated game-file downloads.
- `POST /api/agent/decommission/:id`: cleanup worker reports file removal and
agent uninstall using its own operation-scoped receipt credential.
- `POST /api/game-files`: administrator creates a folder upload with game, name,
version, optional description and `files: [{path, size, lastModified}]`.
- `GET /api/game-files/:id/upload`: durable per-file offsets and publication status.
- `PUT /api/game-files/:id/upload/files/:path`: administrator uploads a raw chunk
with `X-Upload-Offset`, `X-Upload-Length` and `X-Content-SHA256` headers.
- `POST /api/game-files/:id/upload/complete`: administrator starts asynchronous
verification/publication; poll the upload endpoint for completion.
- `/api/agent/libraries/...`: authenticated game-file downloads. Legacy
`/api/agent/imports/...` endpoints remain compatible with already queued imports.
The operation envelope is independent of polling and can be carried over a
persistent transport later. No unrestricted shell operation is available.
@@ -115,17 +135,72 @@ may still join until an eligible empty moment; draining does not kick them.
**Disable** holds agent dispatch. Neither mode, controller downtime nor agent
communication loss stops unrelated running containers.
Administrators can choose **Delete host** on a host card or at the top of its
detail page. The confirmation lists anything that must be resolved first:
registered servers, reserved ports, or operations already sent to the agent.
Drain the host or remove its servers, then let outstanding operations reconcile.
An unenrolled host is removed immediately. An enrolled host stays visible as
**Deleting** until its agent confirms cleanup. Offline hosts wait for their next
connection; their absence is never treated as proof of removal. Existing agents
need the 1.1.0 installer: generate a fresh command under **Agent access**, rerun it
on the host, and wait for a heartbeat advertising uninstall support.
Deletion blocks new operations, cancels queued commands that were never delivered,
and invalidates unused installer links. A separate cleanup service removes the
host's downloaded game libraries, instance files, agent journal and configuration,
owned containers, and recorded runtime images unused by other containers. It
stops and removes the agent service and executable, then removes its own files.
The controller hides the host and revokes its ordinary credentials only after
receiving the final uninstall receipt. Adding the machine again requires a new
host enrollment.
Game folders uploaded through Electric Cherry belong to the controller. Those
original uploads remain available for other hosts and future deployments; only
the deleted host's copies are removed. Controller operation history is also kept.
Unrelated containers, files and shared dependencies such as Docker and Python
are preserved. Historical entries in shared system logs are not erased.
Cleanup retries after connection loss or partial file removal. Failures before
handoff offer **Retry cleanup** after the installation is repaired; failures in
the independent worker retry automatically. The worker stays installed until
the controller acknowledges data cleanup. After removing its final files it
retries the completion receipt from memory. A power loss in that final window
can leave the controller awaiting confirmation even though removal succeeded;
inspect the machine if that status persists. The controller does not guess.
## Libraries and ports
Imports scan the source volume and generate per-file SHA-256 manifests. Uploads
stream to temporary controller storage, are bounded by their manifest sizes and
verified before atomic publication. Published versions cannot be updated;
Choose the actual game folder, not its parent, a ZIP archive or the Plutonium
launcher data folder. The selected folder's name is removed once; nested paths
such as `zone/english/...` and `main/...` stay intact. Plutonium's
[T5 server guide](https://plutonium.pw/docs/server/t5/setting-up-a-server/) and
[T6 server guide](https://plutonium.pw/docs/server/t6/setting-up-a-server/) use a
copy of the game directory, separately from the launcher/runtime. The uploader
preserves the supplied files rather than attempting to infer installed maps from
a manually maintained DLC list.
Folder uploads use chunks of at most 8 MiB, each checked against its SHA-256 before
being appended to temporary controller storage. Memory usage does not grow with
the size of an individual game file. Pause/retry uses the controller's saved file
offsets. After a page reload, choose **Resume upload** and reselect the original,
unchanged folder; relative paths, sizes and modification times must match. Keep
the page open while transferring. Once publication starts, the controller hashes
every complete file and publishes the directory atomically; closing the browser
does not stop that verification. An interrupted controller can retry publication.
Published versions cannot be updated;
changing files requires a new version. Hosts download with HTTP Range resumption,
report progress, verify all checksums and atomically promote the complete cache.
Corrupt or incomplete content never becomes Ready. Cache inventory is reconciled
on heartbeats, including vanished caches. Older immutable versions remain usable
when their exact manifest is verified.
The controller and reverse proxy must accept 8 MiB upload requests and folder
manifests up to 32 MiB. Compose sets `BODY_SIZE_LIMIT=Infinity`; application routes
still enforce these limits, declared file sizes and checksums. Set a proxy body
limit of at least 32 MiB (for example `client_max_body_size 32m` in nginx).
Existing published versions and cached host libraries remain usable after the
upload workflow replaces source-host imports.
Reservations include host ID, local bind address, protocol and port. Wildcard
binds conflict with concrete addresses on the same host. UDP and TCP reservations
for one launch commit together; the same number can be used independently on
@@ -150,7 +225,9 @@ Automated tests cover enrollment/authentication, rotation/revocation, host
readiness, scoped ports, immutable library publication, streaming/ranges,
checksums, interrupted downloads, instance generation ownership, local occupancy
holds, controller restart/reconnect, ambiguous results, occupied campaign staging,
activation before images, collections and draining. Docker/UDP and host
activation before images, collections and draining. Host deletion tests cover
ownership guards, interrupted cleanup, uninstall receipts, credential revocation
and retention of controller uploads/history. Docker/systemd/UDP and host
observations are simulated in these tests.
A real Debian/Docker/Plutonium acceptance run is still required before production
@@ -165,6 +242,10 @@ coverage is not evidence that a real T5/T6 server has been deployed here. Run:
agent activation, and during file activation before restart; verify that the
agent holds disruption.
- Drain occupied lobbies and verify shared libraries survive their retirement.
- Delete a drained host; verify its game copies, data directory, agent and cleanup
services/configuration/executables are removed, controller uploads/history stay,
and unrelated Docker workloads continue. Repeat with connection loss during
cleanup and after the final receipt, and a reboot during data removal.
- Corrupt/interrupted library downloads, resumed transfer and verified promotion.
- Controller restart, agent restart and network loss before execution, during
execution and after execution but before acknowledgement. Verify the same
+2 -2
View File
@@ -10,10 +10,10 @@ export interface HostLibraryCache {
}
export interface GameLibraryVersion {
id: string; libraryId: string; game: GameLibraryGame; version: string;
description: string; dlc: string[]; state: GameLibraryState; sizeBytes: number;
description: string; state: GameLibraryState; sizeBytes: number;
fileCount: number; manifestSha256: string | null; files: GameFileEntry[];
createdAt: string; publishedAt: string | null; storageLocation: string;
sourceHostId: string; sourceVolume: string; error: string | null; operationId: string | null;
sourceHostId: string | null; sourceVolume: string | null; error: string | null; operationId: string | null;
caches: HostLibraryCache[];
}
export interface GameLibrary {
+24 -1
View File
@@ -1,7 +1,28 @@
export type HostMode = 'active' | 'draining' | 'disabled';
export type HostReadiness = 'preparing' | 'ready' | 'degraded' | 'draining' | 'disabled';
export type HostReadiness = 'preparing' | 'needs-runtime' | 'needs-game-files' | 'ready' | 'degraded' | 'draining' | 'disabled' | 'deleting';
export type HostOperationState = 'pending' | 'accepted' | 'running' | 'completed' | 'failed' | 'unknown';
export function hostReadinessLabel(readiness: HostReadiness): string {
return { preparing: 'Preparing', 'needs-runtime': 'Needs runtime', 'needs-game-files': 'Needs game files',
ready: 'Ready', degraded: 'Degraded', draining: 'Draining', disabled: 'Disabled', deleting: 'Deleting' }[readiness];
}
export interface HostDeletion { allowed: boolean; blockers: string[] }
export interface HostDecommission {
operationId: string; stage: 'queued' | 'running' | 'cleaned' | 'failed' | 'completed';
requestedAt: string; updatedAt: string; error: string | null; retryable: boolean;
}
export function hostDeletionMessage(host: Pick<Host, 'connected' | 'decommission'>): string {
if (!host.decommission) return '';
switch (host.decommission.stage) {
case 'queued': return host.connected ? 'Host cleanup is queued.' : 'Waiting for the host to reconnect before removing its files and agent.';
case 'running': return 'Removing host files, cached game libraries, and the agent installation.';
case 'cleaned': return 'Host data is removed. Waiting for confirmation that the agent has uninstalled itself.';
case 'failed': return 'Cleanup needs attention. Complete removal has not been confirmed.';
case 'completed': return 'Host files and agent removal confirmed.';
}
}
export interface HostLibrary { id: string; state: 'missing' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'outdated'; progress?: number; error?: string }
export interface HostInventory {
hostname?: string; agentVersion?: string; protocolVersion?: number; os?: string; architecture?: string;
@@ -9,6 +30,7 @@ export interface HostInventory {
storageTotal?: number; storageAvailable?: number; dockerVersion?: string; dockerRunning?: boolean;
directoriesWritable?: boolean; runtimeImages?: string[]; libraries?: HostLibrary[];
currentOperation?: string | null; [key: string]: unknown;
capabilities?: string[];
}
export interface Host {
id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number;
@@ -16,6 +38,7 @@ export interface Host {
lastHeartbeat: string | null; connected: boolean; connectionState: 'online' | 'offline';
readiness: HostReadiness; readinessReasons: string[]; inventory: HostInventory;
enrolled: boolean; credentialActive: boolean;
decommission: HostDecommission | null;
}
export interface HostOperation {
id: string; hostId: string; kind: string; instanceId: string | null; generation: number | null;
+2 -1
View File
@@ -7,7 +7,8 @@ import { assertRegularPath, parseByteRange } from './storage.js';
export function gameFileError(cause: unknown): Response {
const status = typeof cause === 'object' && cause !== null && 'status' in cause && typeof cause.status === 'number' ? cause.status : 400;
return json({ error: cause instanceof Error ? cause.message : 'Game file request failed.' }, { status });
const receivedBytes = typeof cause === 'object' && cause !== null && 'receivedBytes' in cause && Number.isSafeInteger(cause.receivedBytes) ? cause.receivedBytes : undefined;
return json({ error: cause instanceof Error ? cause.message : 'Game file request failed.', ...(receivedBytes === undefined ? {} : { receivedBytes }) }, { status });
}
export async function readGameJson(request: Request, maxBytes = 64 * 1024): Promise<Record<string, unknown>> {
if (!request.body) throw new Error('A JSON request body is required.');
+112 -8
View File
@@ -6,6 +6,7 @@ import { config } from '../config.js';
import { getHost, getHostOperation, listHosts, queueHostOperation } from '../hosts/index.js';
import { ensureGameFileSchema } from './schema.js';
import { manifestChecksum, receiveGameFile, validateGamePath, validateManifest, verifyGameFile } from './storage.js';
import { MAX_UPLOAD_CHUNK_BYTES, validateUploadPlan, getUploadFileProgress, receiveUploadChunk, hashUploadedFiles, withUploadLock } from './uploads.js';
import type { GameFileEntry, GameLibrary, GameLibraryGame, GameLibraryVersion, HostLibraryCache, LibraryCacheState } from '../../game-files.js';
function database() { const handle = db(); ensureGameFileSchema(handle); return handle; }
@@ -17,10 +18,10 @@ const cacheFromRow = (row: CacheRow): HostLibraryCache => ({ hostId: row.host_id
function versionFromRow(row: VersionRow): GameLibraryVersion {
const files = JSON.parse(row.manifest) as GameFileEntry[];
return { id: row.id, libraryId: row.library_id, game: row.game, version: row.version,
description: row.description, dlc: JSON.parse(row.dlc), state: row.state, sizeBytes: row.size_bytes,
files, fileCount: files.length, manifestSha256: row.manifest_sha256, createdAt: row.created_at,
publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id,
sourceVolume: row.source_volume, error: row.error, operationId: row.operation_id,
description: row.description, state: row.state, sizeBytes: row.size_bytes,
files, fileCount: files.length || (database().prepare('SELECT COUNT(*) AS count FROM game_file_upload_entries WHERE version_id=?').get(row.id) as { count: number }).count, manifestSha256: row.manifest_sha256, createdAt: row.created_at,
publishedAt: row.published_at, storageLocation: row.storage_location, sourceHostId: row.source_host_id || null,
sourceVolume: row.source_volume || null, error: row.error, operationId: row.operation_id,
caches: (database().prepare('SELECT * FROM host_library_caches WHERE version_id = ?').all(row.id) as CacheRow[]).map(cacheFromRow) };
}
export function getLibraryVersion(id: string): GameLibraryVersion | null {
@@ -60,6 +61,112 @@ function textField(input: unknown, label: string, max: number, required = true):
if (typeof input !== 'string' || input.length > max || (required && !input.trim())) throw new Error(`Provide a valid ${label}.`);
return input.trim();
}
/** Register a folder on the controller; no host or Docker volume is needed. */
export function createLibraryUpload(input: Record<string, unknown>): GameLibraryVersion {
const game = input.game;
if (game !== 't5' && game !== 't6') throw new Error('Choose T5 or T6 game files.');
const name = textField(input.name, 'library name', 160);
const description = textField(input.description ?? '', 'description', 4000, false);
const version = textField(input.version, 'version label', 120);
const files = validateUploadPlan(input.files);
const handle = database();
const id = randomUUID(); const now = new Date().toISOString();
handle.transaction(() => {
let library = handle.prepare('SELECT id,game FROM game_file_libraries WHERE name=?').get(name) as { id: string; game: string } | undefined;
if (library && library.game !== game) throw new Error('This library name is already used for another game.');
if (!library) {
library = { id: randomUUID(), game };
handle.prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)').run(library.id, name, game, description, now);
}
if (handle.prepare('SELECT 1 FROM game_file_versions WHERE library_id=? AND version=?').get(library.id, version)) throw new Error('This version label already exists. Choose a new version.');
// Keep legacy non-null columns for existing databases and published versions.
handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,size_bytes,created_at,storage_location,source_host_id,source_volume)
VALUES(?,?,?,?,'[]','importing',?,?,?,'','')`).run(id, library.id, version, description, files.reduce((sum, file) => sum + file.size, 0), now, join(gameFilesRoot(), game, id));
const insert = handle.prepare('INSERT INTO game_file_upload_entries(version_id,path,size,last_modified) VALUES(?,?,?,?)');
for (const file of files) insert.run(id, file.path, file.size, file.lastModified);
})();
return getLibraryVersion(id)!;
}
type UploadEntry = { path: string; size: number; lastModified: number };
function requireBrowserUpload(id: string): GameLibraryVersion {
const version = getLibraryVersion(id);
if (!version || version.sourceHostId !== null || !database().prepare('SELECT 1 FROM game_file_upload_entries WHERE version_id=? LIMIT 1').get(id)) {
throw Object.assign(new Error('Folder upload not found.'), { status: 404 });
}
return version;
}
function uploadEntries(id: string): UploadEntry[] {
return database().prepare('SELECT path,size,last_modified AS lastModified FROM game_file_upload_entries WHERE version_id=? ORDER BY path').all(id) as UploadEntry[];
}
async function uploadStorageRoot(version: GameLibraryVersion): Promise<string> {
// Recover publication interrupted after the atomic rename but before its DB commit.
try { if ((await stat(version.storageLocation)).isDirectory()) return version.storageLocation; }
catch (cause) { if ((cause as NodeJS.ErrnoException).code !== 'ENOENT') throw cause; }
return stagingRoot(version.id);
}
export async function browserUploadStatus(id: string) {
const version = requireBrowserUpload(id);
const status = version.state === 'ready' ? 'published' as const : publishing.has(id) ? 'verifying' as const : version.state === 'failed' ? 'failed' as const : 'pending' as const;
const root = await uploadStorageRoot(version);
const entries = uploadEntries(id);
const files = [];
// Avoid opening thousands of files concurrently for a large folder.
for (const file of entries) files.push({ ...file, ...(status === 'published' ? { receivedBytes: file.size, complete: true } : await getUploadFileProgress(root, file)) });
const { files: _manifest, ...summary } = version;
return { status, chunkSize: MAX_UPLOAD_CHUNK_BYTES, files, version: summary, error: version.error };
}
export async function uploadBrowserFile(id: string, path: string, chunk: { offset: number; size: number; sha256: string }, body: ReadableStream<Uint8Array> | null) {
validateGamePath(path);
const checkWritable = () => {
const version = requireBrowserUpload(id);
if (version.state === 'ready' || version.manifestSha256 || publishing.has(id)) throw Object.assign(new Error('This upload is no longer writable.'), { status: 409 });
};
checkWritable();
const file = database().prepare('SELECT path,size,last_modified AS lastModified FROM game_file_upload_entries WHERE version_id=? AND path=?').get(id, path) as UploadEntry | undefined;
if (!file) throw new Error('File is not listed in this upload.');
const progress = await receiveUploadChunk(stagingRoot(id), file, chunk, body, checkWritable);
database().prepare("UPDATE game_file_versions SET state='importing',error=NULL WHERE id=? AND state IN ('importing','failed')").run(id);
return progress;
}
export async function publishBrowserUpload(id: string): Promise<GameLibraryVersion> {
const version = requireBrowserUpload(id);
if (version.state === 'ready') return version;
const existing = publishing.get(id); if (existing) return existing;
const task = withUploadLock(stagingRoot(id), async () => {
const root = await uploadStorageRoot(version);
const files = await hashUploadedFiles(root, uploadEntries(id));
const checksum = manifestChecksum(files);
if (version.manifestSha256 && checksum !== version.manifestSha256) throw new Error('Stored files no longer match the verified manifest.');
database().transaction(() => {
database().prepare("UPDATE game_file_versions SET state='importing',manifest=?,manifest_sha256=?,error=NULL WHERE id=? AND state IN ('importing','failed')").run(JSON.stringify(files), checksum, id);
database().prepare('DELETE FROM game_file_entries WHERE version_id=?').run(id);
const insert = database().prepare('INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)');
for (const file of files) insert.run(id, file.path, file.size, file.sha256);
})();
if (root !== version.storageLocation) {
await mkdir(dirname(version.storageLocation), { recursive: true });
await rename(root, version.storageLocation);
}
database().prepare("UPDATE game_file_versions SET state='ready',published_at=?,error=NULL WHERE id=? AND state='importing'").run(new Date().toISOString(), id);
return getLibraryVersion(id)!;
});
publishing.set(id, task);
try { return await task; } finally { publishing.delete(id); }
}
export async function requestBrowserPublication(id: string) {
const version = requireBrowserUpload(id);
if (version.state !== 'ready' && !publishing.has(id)) {
database().prepare("UPDATE game_file_versions SET state='importing',error=NULL WHERE id=? AND state IN ('importing','failed')").run(id);
void publishBrowserUpload(id).catch(cause => {
database().prepare("UPDATE game_file_versions SET state='failed',error=? WHERE id=? AND state='importing'")
.run((cause instanceof Error ? cause.message : 'Folder publication failed.').slice(0, 2000), id);
});
}
return browserUploadStatus(id);
}
export function createLibraryImport(input: Record<string, unknown>): GameLibraryVersion {
const game = input.game;
if (game !== 't5' && game !== 't6') throw new Error('Choose T5 or T6 game files.');
@@ -69,9 +176,6 @@ export function createLibraryImport(input: Record<string, unknown>): GameLibrary
const name = textField(input.name, 'library name', 160);
const description = textField(input.description ?? '', 'description', 4000, false);
const version = textField(input.version, 'version label', 120);
const dlcInput = input.dlc ?? [];
if (!Array.isArray(dlcInput) || dlcInput.length > 64) throw new Error('DLC must be a list of content names.');
const dlc = dlcInput.map(value => textField(value, 'DLC name', 120));
const sourceVolume = game === 't5' ? 't5_game_files' : 't6_game_files';
const handle = database();
let library = handle.prepare('SELECT id,game FROM game_file_libraries WHERE name=?').get(name) as { id: string; game: string } | undefined;
@@ -84,7 +188,7 @@ export function createLibraryImport(input: Record<string, unknown>): GameLibrary
}
if (handle.prepare('SELECT 1 FROM game_file_versions WHERE library_id=? AND version=?').get(library.id, version)) throw new Error('This version label already exists. Choose a new version.');
handle.prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume)
VALUES(?,?,?,?,?,'importing',?,?,?,?)`).run(id, library.id, version, description, JSON.stringify(dlc), now, join(gameFilesRoot(), game, id), hostId, sourceVolume);
VALUES(?,?,?,?,?,'importing',?,?,?,?)`).run(id, library.id, version, description, '[]', now, join(gameFilesRoot(), game, id), hostId, sourceVolume);
})();
try {
const operation = queueHostOperation(hostId, 'library.import', {
+5
View File
@@ -29,6 +29,11 @@ export function ensureGameFileSchema(handle: Database.Database): void {
version_id TEXT NOT NULL REFERENCES game_file_versions(id), path TEXT NOT NULL,
size INTEGER NOT NULL, sha256 TEXT NOT NULL, PRIMARY KEY(version_id,path)
);
CREATE TABLE IF NOT EXISTS game_file_upload_entries (
version_id TEXT NOT NULL REFERENCES game_file_versions(id), path TEXT NOT NULL,
size INTEGER NOT NULL, last_modified INTEGER NOT NULL,
PRIMARY KEY(version_id,path)
);
INSERT OR IGNORE INTO game_file_entries(version_id,path,size,sha256)
SELECT v.id,json_extract(f.value,'$.path'),json_extract(f.value,'$.size'),json_extract(f.value,'$.sha256')
FROM game_file_versions v,json_each(v.manifest) f;
+7 -2
View File
@@ -63,13 +63,18 @@ export async function assertRegularPath(root: string, path: string, allowMissing
return result;
}
export async function verifyGameFile(root: string, file: GameFileEntry): Promise<void> {
/** Compute a file digest without buffering game archives in memory. */
export async function hashGameFile(root: string, file: Pick<GameFileEntry, 'path' | 'size'>): Promise<string> {
const path = await assertRegularPath(root, file.path);
const info = await stat(path);
if (info.size !== file.size) throw new Error(`Size verification failed for ${file.path}.`);
const hash = createHash('sha256');
for await (const chunk of createReadStream(path)) hash.update(chunk);
if (hash.digest('hex') !== file.sha256) throw new Error(`Checksum verification failed for ${file.path}.`);
return hash.digest('hex');
}
export async function verifyGameFile(root: string, file: GameFileEntry): Promise<void> {
if (await hashGameFile(root, file) !== file.sha256) throw new Error(`Checksum verification failed for ${file.path}.`);
}
/** Stream to a private temporary file; publish only after byte count and SHA256 match. */
+155
View File
@@ -0,0 +1,155 @@
import { createHash, randomUUID } from 'node:crypto';
import { constants, createReadStream, createWriteStream } from 'node:fs';
import { lstat, mkdir, open, rm } from 'node:fs/promises';
import { dirname, resolve } from 'node:path';
import { Readable, Transform } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import type { GameFileEntry } from '../../game-files.js';
import { assertRegularPath, hashGameFile, validateManifest } from './storage.js';
export const MAX_UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024;
export interface UploadFilePlan {
path: string;
size: number;
lastModified: number;
}
export interface UploadFileProgress {
receivedBytes: number;
complete: boolean;
}
export class UploadOffsetError extends Error {
readonly status = 409;
readonly receivedBytes: number;
constructor(receivedBytes: number) {
super(`Upload offset does not match the stored file. Resume at byte ${receivedBytes}.`);
this.name = 'UploadOffsetError';
this.receivedBytes = receivedBytes;
}
}
/** Apply the same path, case, size and file/directory collision rules as published manifests. */
export function validateUploadPlan(input: unknown): UploadFilePlan[] {
if (!Array.isArray(input) || !input.length || input.length > 200_000) throw new Error('Provide a folder containing 1–200000 files.');
const modified = new Map<string, number>();
const manifest = validateManifest(input.map(raw => {
if (!raw || typeof raw !== 'object') throw new Error('Invalid upload file.');
const lastModified = raw.lastModified === undefined ? 0 : raw.lastModified;
if (!Number.isSafeInteger(lastModified) || lastModified < 0) throw new Error('Invalid file modification time.');
modified.set(raw.path, lastModified);
return { path: raw.path, size: raw.size, sha256: '0'.repeat(64) };
}));
return manifest.map(file => ({ path: file.path, size: file.size, lastModified: modified.get(file.path)! }));
}
const uploadLocks = new Map<string, Promise<void>>();
/** Serialize transfers and final publication for one import; a rejected operation never poisons the lock. */
export async function withUploadLock<T>(root: string, callback: () => Promise<T> | T): Promise<T> {
const key = resolve(root);
const previous = uploadLocks.get(key) ?? Promise.resolve();
let release!: () => void;
const current = new Promise<void>(done => { release = done; });
uploadLocks.set(key, current);
await previous;
try { return await callback(); }
finally {
release();
if (uploadLocks.get(key) === current) uploadLocks.delete(key);
}
}
/** File size is the resume checkpoint, including after a controller restart during an append. */
export async function getUploadFileProgress(root: string, file: Pick<UploadFilePlan, 'path' | 'size'>): Promise<UploadFileProgress> {
const path = await assertRegularPath(root, file.path, true);
try {
const info = await lstat(path);
if (!info.isFile() || info.isSymbolicLink()) throw new Error('Game storage contains a non-regular path.');
if (info.size > file.size) throw new Error(`Stored upload exceeds the declared size of ${file.path}.`);
return { receivedBytes: info.size, complete: info.size === file.size };
} catch (cause) {
if ((cause as NodeJS.ErrnoException).code === 'ENOENT') return { receivedBytes: 0, complete: false };
throw cause;
}
}
/**
* Verify one bounded request before touching the destination. Only verified bytes may be
* appended, so a process interruption during the append can resume at the actual file size.
* beforeWrite runs under the lock: callers must recheck that the import is still writable.
*/
export async function receiveUploadChunk(
root: string,
file: Pick<UploadFilePlan, 'path' | 'size'>,
chunk: { offset: number; size: number; sha256: string },
body: ReadableStream<Uint8Array> | null,
beforeWrite?: () => Promise<void> | void
): Promise<UploadFileProgress> {
if (!Number.isSafeInteger(chunk.offset) || chunk.offset < 0 || !Number.isSafeInteger(chunk.size)
|| chunk.size < 0 || chunk.size > MAX_UPLOAD_CHUNK_BYTES || chunk.offset + chunk.size > file.size
|| !Number.isSafeInteger(chunk.offset + chunk.size) || !/^[a-f0-9]{64}$/.test(chunk.sha256)
|| (chunk.size === 0 && file.size !== 0)) {
throw new Error('Invalid upload chunk size, offset or SHA256.');
}
if (!body && chunk.size !== 0) throw new Error('A file chunk body is required.');
return withUploadLock(root, async () => {
await beforeWrite?.();
const progress = await getUploadFileProgress(root, file);
if (chunk.offset !== progress.receivedBytes) throw new UploadOffsetError(progress.receivedBytes);
const destination = await assertRegularPath(root, file.path, true);
// Keep transient chunks outside the tree that will become the immutable library.
const chunkRoot = `${resolve(root)}.chunks`;
const temporary = await assertRegularPath(chunkRoot, randomUUID(), true);
await mkdir(dirname(temporary), { recursive: true });
let received = 0;
const hash = createHash('sha256');
const verifier = new Transform({ transform(data, _encoding, done) {
received += data.length;
if (received > chunk.size) return done(new Error('Upload exceeds the declared chunk size.'));
hash.update(data); done(null, data);
} });
try {
await pipeline(body ? Readable.fromWeb(body as never) : Readable.from([]), verifier,
createWriteStream(temporary, { flags: 'wx', mode: 0o600, flush: true }));
if (received !== chunk.size || hash.digest('hex') !== chunk.sha256) throw new Error('Upload chunk checksum or size mismatch.');
await mkdir(dirname(destination), { recursive: true });
await assertRegularPath(root, file.path, true);
const target = await open(destination, constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW, 0o600);
try {
const info = await target.stat();
if (!info.isFile()) throw new Error('Game storage contains a non-regular path.');
if (info.size !== chunk.offset) throw new UploadOffsetError(info.size);
let position = chunk.offset;
try {
for await (const data of createReadStream(temporary)) {
const bytes = data as Buffer;
let written = 0;
while (written < bytes.length) {
const result = await target.write(bytes, written, bytes.length - written, position);
if (result.bytesWritten === 0) throw new Error('Unable to append the upload chunk.');
written += result.bytesWritten;
position += result.bytesWritten;
}
}
await target.sync();
} catch (cause) {
await target.truncate(chunk.offset);
await target.sync();
throw cause;
}
} finally { await target.close(); }
return { receivedBytes: chunk.offset + chunk.size, complete: chunk.offset + chunk.size === file.size };
} finally { await rm(temporary, { force: true }); }
});
}
/** Caller holds withUploadLock through hashing and publication to prevent concurrent writes. */
export async function hashUploadedFiles(root: string, files: ReadonlyArray<Pick<UploadFilePlan, 'path' | 'size'>>): Promise<GameFileEntry[]> {
const manifest: GameFileEntry[] = [];
for (const file of files) manifest.push({ path: file.path, size: file.size, sha256: await hashGameFile(root, file) });
return validateManifest(manifest);
}
+133 -10
View File
@@ -1,8 +1,8 @@
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
import { isIP } from 'node:net';
import { db } from '../db/index.js';
import { ensureHostSchema } from './schema.js';
import type { Host, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
export const AGENT_PROTOCOL_VERSION = 1;
export const HEARTBEAT_MAX_AGE_MS = 90_000;
@@ -11,6 +11,14 @@ const digest = (token: string) => createHash('sha256').update(token).digest('hex
function database() { const handle = db(); ensureHostSchema(handle); return handle; }
type HostRow = { id: string; name: string; hostname: string; mode: HostMode; port_min: number; port_max: number; bind_address: string; public_address: string; minimum_free_bytes: number; created_at: string; enrolled_at: string | null; last_heartbeat: string | null; inventory: string; credential_active: number };
const hostSql = `SELECT h.*, EXISTS(SELECT 1 FROM ec_host_credentials c WHERE c.host_id = h.id AND c.revoked_at IS NULL) AS credential_active FROM ec_hosts h`;
type DecommissionRow = { host_id: string; operation_id: string; token_hash: string; stage: HostDecommission['stage']; requested_at: string; updated_at: string; cleaned_at: string | null; error: string | null };
function decommissionRow(hostId: string): DecommissionRow | undefined {
return database().prepare('SELECT * FROM ec_host_decommissions WHERE host_id=?').get(hostId) as DecommissionRow | undefined;
}
function presentDecommission(row: DecommissionRow | undefined): HostDecommission | null {
return row ? { operationId: row.operation_id, stage: row.stage, requestedAt: row.requested_at, updatedAt: row.updated_at,
error: row.error, retryable: row.stage === 'failed' && getHostOperation(row.operation_id)?.state === 'failed' } : null;
}
function hostFromRow(row: HostRow): Host {
const inventory: HostInventory = JSON.parse(row.inventory);
const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
@@ -23,22 +31,123 @@ function hostFromRow(row: HostRow): Host {
if (!inventory.directoriesWritable) reasons.push('Managed directories are not writable.');
if (typeof inventory.storageAvailable !== 'number' || inventory.storageAvailable < row.minimum_free_bytes) reasons.push('Insufficient available game storage.');
const infrastructureReady = reasons.length === 0;
if (!inventory.runtimeImages?.length) reasons.push('Prepare a Plutainer runtime image.');
if (!inventory.libraries?.some(library => library.state === 'ready')) reasons.push('Download and verify a game-file library.');
const runtimeReady = !!inventory.runtimeImages?.length;
const libraryReady = !!inventory.libraries?.some(library => library.state === 'ready');
if (!runtimeReady) reasons.push('Prepare a Plutainer runtime image.');
if (!libraryReady) reasons.push('Download and verify a game-file library.');
if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
if (row.mode === 'disabled') reasons.push('Host is disabled.');
const decommission = presentDecommission(decommissionRow(row.id));
if (decommission) reasons.push('Host deletion is in progress; waiting for confirmed cleanup and agent removal.');
return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max,
bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes,
createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline',
readiness: row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at || (infrastructureReady && reasons.length) ? 'preparing' : reasons.length ? 'degraded' : 'ready',
readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active };
readiness: decommission ? 'deleting' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active, decommission };
}
export function listHosts(): Host[] { return (database().prepare(hostSql + ' ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
export function getHost(id: string): Host | null {
const row = database().prepare(hostSql + ' WHERE h.id = ?').get(id) as HostRow | undefined;
const row = database().prepare(hostSql + ' WHERE h.id = ? AND h.deleted_at IS NULL').get(id) as HostRow | undefined;
return row ? hostFromRow(row) : null;
}
function requireHost(id: string): Host { const host = getHost(id); if (!host) throw new Error('Host was not found.'); return host; }
export class HostDeletionError extends Error {
constructor(message: string, readonly status: 400 | 401 | 404 | 409, readonly blockers: string[] = []) { super(message); }
}
/** Controller records can be removed only after all remote instance ownership is resolved. */
export function hostDeletionStatus(id: string): HostDeletion {
const host = getHost(id);
if (!host) throw new HostDeletionError('Host not found.', 404);
if (host.decommission && !host.decommission.retryable) return { allowed: true, blockers: [] };
const handle = database();
const blockers: string[] = [];
if (host.enrolled && !host.inventory.capabilities?.includes('host.decommission')) blockers.push('Update this host agent before deleting it: generate an installer from Agent access, rerun it on this host, and wait for a fresh heartbeat.');
if (host.enrolled && !host.credentialActive) blockers.push('Re-enroll this host so its agent can receive and complete the cleanup request.');
const instances = handle.prepare('SELECT server FROM native_instances WHERE host_id=? AND retired_at IS NULL ORDER BY server').all(id) as { server: string }[];
if (instances.length) {
const names = instances.slice(0, 5).map(instance => instance.server).join(', ') + (instances.length > 5 ? ', …' : '');
blockers.push(`Remove the ${instances.length} server instance${instances.length === 1 ? '' : 's'} assigned to this host first (${names}). Queued and failed deployments also require removal.`);
}
const ports = handle.prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE host_id=?').get(id) as { count: number };
if (ports.count && !instances.length) blockers.push(`This host still has ${ports.count} reserved game port${ports.count === 1 ? '' : 's'}. Finish or reconcile the corresponding server removals first.`);
// A pending command may have reached the agent before its acknowledgement.
// Only commands that have never been delivered are safe to cancel locally.
const operations = handle.prepare(`SELECT kind,state FROM ec_host_operations WHERE host_id=?
AND (state IN ('accepted','running','unknown') OR (state='pending' AND last_delivered_at IS NOT NULL))
ORDER BY created_at`).all(id) as { kind: string; state: string }[];
if (operations.length) {
const work = operations.slice(0, 5).map(operation => `${operation.kind}: ${operation.state}`).join(', ') + (operations.length > 5 ? ', …' : '');
blockers.push(`${operations.length} delivered operation${operations.length === 1 ? ' is' : 's are'} unresolved (${work}). Let the agent reconnect and finish or reconcile this work before deleting the host.`);
}
return { allowed: blockers.length === 0, blockers };
}
/** Keep the host visible and authenticated until its independent cleanup worker proves removal. */
export function deleteHost(id: string): { deleted: boolean; operationId?: string } {
const handle = database();
return handle.transaction(() => {
const host = getHost(id);
if (!host) throw new HostDeletionError('Host not found.', 404);
if (host.decommission && !host.decommission.retryable) return { deleted: false, operationId: host.decommission.operationId };
const status = hostDeletionStatus(id);
if (!status.allowed) throw new HostDeletionError(status.blockers.join(' '), 409, status.blockers);
const timestamp = now();
const reason = 'Host deletion requested before this operation was delivered to the agent.';
handle.prepare(`INSERT INTO ec_host_operation_events(operation_id,state,created_at,error)
SELECT id,'failed',?,? FROM ec_host_operations WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL`).run(timestamp, reason, id);
handle.prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE host_id=? AND state='pending' AND last_delivered_at IS NULL").run(reason, timestamp, id);
handle.prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, id);
if (!host.enrolled) {
handle.prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=? AND deleted_at IS NULL").run(timestamp, id);
return { deleted: true };
}
const operationId = randomUUID();
const token = 'ec_cleanup_' + randomBytes(32).toString('base64url');
const payload = { hostId: id, operationId, receiptToken: token, receiptUrl: `/api/agent/decommission/${operationId}` };
handle.prepare("INSERT INTO ec_host_operations(id,host_id,kind,payload,state,created_at,updated_at) VALUES(?,?,'host.decommission',?,'pending',?,?)").run(operationId, id, JSON.stringify(payload), timestamp, timestamp);
handle.prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'pending',?)").run(operationId, timestamp);
handle.prepare(`INSERT INTO ec_host_decommissions(host_id,operation_id,token_hash,requested_at,updated_at) VALUES(?,?,?,?,?)
ON CONFLICT(host_id) DO UPDATE SET operation_id=excluded.operation_id,token_hash=excluded.token_hash,stage='queued',requested_at=excluded.requested_at,updated_at=excluded.updated_at,cleaned_at=NULL,error=NULL`)
.run(id, operationId, digest(token), timestamp, timestamp);
handle.prepare("UPDATE ec_hosts SET mode='draining' WHERE id=?").run(id);
return { deleted: false, operationId };
}).immediate();
}
export function receiveDecommissionReceipt(operationId: string, token: string, input: Record<string, unknown>): { acknowledged: true } {
if (!token || token.length > 200) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
return database().transaction(() => {
const row = database().prepare('SELECT * FROM ec_host_decommissions WHERE operation_id=?').get(operationId) as DecommissionRow | undefined;
if (!row || !timingSafeEqual(Buffer.from(row.token_hash, 'hex'), Buffer.from(digest(token), 'hex'))) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
if (!['cleaned', 'completed', 'failed'].includes(input.stage as string)) throw new HostDeletionError('Invalid cleanup receipt stage.', 400);
if (row.stage === 'completed') return { acknowledged: true as const };
const operation = getHostOperation(operationId)!;
if (!['accepted','running','unknown'].includes(operation.state)) throw new HostDeletionError('Cleanup has not been accepted by the host.', 409);
const timestamp = now();
if (input.stage === 'failed') {
database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE host_id=?")
.run(typeof input.error === 'string' ? input.error.slice(0, 2000) : 'Host cleanup needs attention. The cleanup worker will retry.', timestamp, row.host_id);
return { acknowledged: true as const };
}
if (input.stage === 'cleaned') {
database().prepare("UPDATE ec_host_decommissions SET stage='cleaned',cleaned_at=COALESCE(cleaned_at,?),error=NULL,updated_at=? WHERE host_id=?").run(timestamp, timestamp, row.host_id);
return { acknowledged: true as const };
}
if (!row.cleaned_at) throw new HostDeletionError('The host must confirm file cleanup before confirming agent removal.', 409);
database().prepare("UPDATE ec_host_decommissions SET stage='completed',error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id);
database().prepare("UPDATE ec_host_operations SET state='completed',result=?,error=NULL,updated_at=? WHERE id=?").run(JSON.stringify({ filesRemoved: true, agentRemoved: true }), timestamp, operationId);
database().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,'completed',?)").run(operationId, timestamp);
database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(timestamp, row.host_id);
database().prepare('UPDATE ec_host_enrollments SET consumed_at=? WHERE host_id=? AND consumed_at IS NULL').run(timestamp, row.host_id);
if (database().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='host_library_caches'").get()) {
database().prepare("UPDATE host_library_caches SET status='missing',bytes_downloaded=0,manifest_sha256=NULL,error=NULL,updated_at=? WHERE host_id=?").run(timestamp, row.host_id);
}
database().prepare("UPDATE ec_hosts SET deleted_at=?,mode='disabled' WHERE id=?").run(timestamp, row.host_id);
return { acknowledged: true as const };
}).immediate();
}
function textField(value: unknown, label: string, max = 200): string {
if (typeof value !== 'string' || !value.trim() || value.length > max || /[\x00-\x1f]/.test(value)) throw new Error(`${label} must be a nonempty string up to ${max} characters.`);
return value.trim();
@@ -55,6 +164,7 @@ export function createHost(input: Record<string, unknown>): Host {
}
export function updateHost(id: string, input: Record<string, unknown>): Host {
const host = requireHost(id);
if (host.decommission) throw new Error('Host cleanup is in progress; its settings cannot change.');
const name = input.name === undefined ? host.name : textField(input.name, 'Host name');
const mode = input.mode ?? host.mode;
if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.');
@@ -70,7 +180,8 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
}
export function createEnrollment(hostId: string, ttlMs = 15 * 60_000): { id: string; hostId: string; token: string; expiresAt: string } {
requireHost(hostId);
const host = requireHost(hostId);
if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup is in progress; enrollment is unavailable.');
if (!Number.isInteger(ttlMs) || ttlMs <= 0 || ttlMs > 60 * 60_000) throw new Error('Enrollment lifetime must be at most one hour.');
const token = 'ec_enroll_' + randomBytes(32).toString('base64url'); const id = randomUUID();
const expiresAt = new Date(Date.now() + ttlMs).toISOString();
@@ -96,6 +207,7 @@ export function validateInventory(value: unknown): HostInventory {
}
for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
if (inventory.libraries !== undefined && (!Array.isArray(inventory.libraries) || inventory.libraries.length > 2000 || inventory.libraries.some(library =>
!library || typeof library !== 'object' || typeof library.id !== 'string' || !library.id || library.id.length > 200 || !['missing','downloading','verifying','ready','failed','outdated'].includes(library.state)))) throw new Error('Invalid game library inventory.');
return inventory;
@@ -122,7 +234,8 @@ export function authenticateCredential(token: string): Host | null {
return getHost(row.host_id);
}
export function revokeCredentials(hostId: string): void {
requireHost(hostId);
const host = requireHost(hostId);
if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup needs its credentials until removal is confirmed.');
database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(now(), hostId);
}
export function rotateCredential(hostId: string, nextCredential?: string): string {
@@ -160,6 +273,8 @@ export function getHostOperation(id: string): HostOperation | null { const row =
export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation {
const host = requireHost(hostId);
if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.');
if (kind === 'host.decommission') throw new Error('Use the host deletion workflow to request cleanup.');
if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.');
@@ -195,8 +310,10 @@ export function pendingHostOperations(hostId: string): HostOperation[] {
}
export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation {
return database().transaction(() => {
requireHost(hostId);
const operation = getHostOperation(id);
if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.');
if (operation.kind === 'host.decommission' && input.state === 'completed') throw new Error('Host removal requires the cleanup worker completion receipt.');
if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.');
if (['completed','failed'].includes(operation.state)) return operation;
const state = input.state as HostOperationState;
@@ -205,6 +322,10 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
const result = input.result === undefined ? null : JSON.stringify(input.result);
if (result !== null && Buffer.byteLength(result) > 128 * 1024 * 1024) throw new Error('Operation result exceeds the limit.');
database().prepare('UPDATE ec_host_operations SET state=?,result=?,error=?,updated_at=? WHERE id=?').run(state, result, error, now(), id);
if (operation.kind === 'host.decommission') {
if (state === 'failed') database().prepare("UPDATE ec_host_decommissions SET stage='failed',error=?,updated_at=? WHERE operation_id=?").run(error ?? 'The agent could not start cleanup. Repair the installation, then retry deletion.', now(), id);
else database().prepare("UPDATE ec_host_decommissions SET stage='running',updated_at=? WHERE operation_id=? AND stage IN ('queued','running')").run(now(), id);
}
if (operation.state !== state) database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,?,?,?)').run(id, state, now(), error);
return getHostOperation(id)!;
})();
@@ -222,6 +343,7 @@ export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise
export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean {
const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress;
if (host.decommission) return false;
if (candidate < host.portMin || candidate > host.portMax) return false;
const rows = database().prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND protocol=? AND port=?').all(hostId, protocol, candidate) as { instance_id: string; bind_address: string }[];
return !rows.some(row => row.instance_id !== instanceId && (row.bind_address === bind || ['0.0.0.0','::'].includes(bind) || ['0.0.0.0','::'].includes(row.bind_address)));
@@ -233,6 +355,7 @@ export function nextHostPort(hostId: string, additionalPorts: number[] = [], bin
}
export function reserveHostPort(hostId: string, instanceId: string, options: { port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): number {
const host = requireHost(hostId); const bind = options.bindAddress ?? host.bindAddress; const protocol = options.protocol ?? 'udp';
if (host.decommission) throw new Error('Host cleanup is in progress; new port reservations are blocked.');
if (!isIP(bind)) throw new Error('Bind address must be an IP address.');
if (!['udp','tcp'].includes(protocol)) throw new Error('Port protocol must be UDP or TCP.');
if (!instanceId) throw new Error('An immutable instance ID is required.');
+10 -1
View File
@@ -10,7 +10,7 @@ export function ensureHostSchema(handle: Database.Database): void {
port_min INTEGER NOT NULL DEFAULT 28960, port_max INTEGER NOT NULL DEFAULT 28980,
bind_address TEXT NOT NULL DEFAULT '0.0.0.0', public_address TEXT NOT NULL DEFAULT '',
minimum_free_bytes INTEGER NOT NULL DEFAULT 1073741824,
created_at TEXT NOT NULL, enrolled_at TEXT, last_heartbeat TEXT, inventory TEXT NOT NULL DEFAULT '{}'
created_at TEXT NOT NULL, enrolled_at TEXT, last_heartbeat TEXT, inventory TEXT NOT NULL DEFAULT '{}', deleted_at TEXT
);
CREATE TABLE IF NOT EXISTS ec_host_enrollments (
id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), token_hash TEXT NOT NULL UNIQUE,
@@ -40,8 +40,17 @@ export function ensureHostSchema(handle: Database.Database): void {
PRIMARY KEY(host_id,bind_address,protocol,port)
);
CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id);
CREATE TABLE IF NOT EXISTS ec_host_decommissions (
host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id),
operation_id TEXT NOT NULL UNIQUE REFERENCES ec_host_operations(id),
token_hash TEXT NOT NULL, stage TEXT NOT NULL DEFAULT 'queued'
CHECK(stage IN ('queued','running','cleaned','failed','completed')),
requested_at TEXT NOT NULL, updated_at TEXT NOT NULL, cleaned_at TEXT, error TEXT
);
`);
const columns = handle.prepare('PRAGMA table_info(ec_host_operations)').all() as { name: string }[];
if (!columns.some(column => column.name === 'last_delivered_at')) handle.exec('ALTER TABLE ec_host_operations ADD COLUMN last_delivered_at TEXT');
const hostColumns = handle.prepare('PRAGMA table_info(ec_hosts)').all() as { name: string }[];
if (!hostColumns.some(column => column.name === 'deleted_at')) handle.exec('ALTER TABLE ec_hosts ADD COLUMN deleted_at TEXT');
initialized.add(handle);
}
+1 -1
View File
@@ -204,7 +204,7 @@ export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blue
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All 20 lobbies for this map are occupied or reserved.' },
{ label: 'Game port', state: port === null ? 'blocked' : 'pass', detail: port === null ? 'No free game port. Free capacity before launching.' : `${port} / UDP and TCP. Availability is checked again at launch.` },
{ label: 'Promoted configuration', state: 'pass', detail: recipe.configSource },
{ label: 'Game files and host readiness', state: 'info', detail: 'The host must have the selected game and DLC files. This preview does not probe volumes, images, credentials, or network access.' }
{ label: 'Game files and host readiness', state: 'info', detail: 'The host must have the selected game files. This preview does not probe volumes, images, credentials, or network access.' }
];
const id = randomUUID();
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
@@ -0,0 +1,66 @@
<script lang="ts">
import type { Host, HostDeletion } from '$lib/hosts.js';
import Button from './primitives/Button.svelte';
import Dialog from './primitives/Dialog.svelte';
let { open = $bindable(false), host, ondeleted }:
{ open?: boolean; host: Pick<Host, 'id' | 'name'> | null; ondeleted: (result: { deleted: boolean; operationId?: string }) => void | Promise<void> } = $props();
let deletion = $state<HostDeletion | null>(null);
let checking = $state(false); let deleting = $state(false); let error = $state('');
async function inspect(id: string, signal?: AbortSignal) {
checking = true; deletion = null; error = '';
try {
const response = await fetch(`/api/hosts/${encodeURIComponent(id)}`, { signal });
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not check this host.');
if (!signal?.aborted) deletion = body.deletion;
} catch (cause) { if (!signal?.aborted) error = (cause as Error).message; }
finally { if (!signal?.aborted) checking = false; }
}
$effect(() => {
if (!open || !host) return;
const controller = new AbortController();
void inspect(host.id, controller.signal);
return () => controller.abort();
});
async function remove() {
if (!host || deleting || checking || !deletion?.allowed) return;
const id = host.id;
deleting = true; error = '';
try {
const response = await fetch(`/api/hosts/${encodeURIComponent(id)}`, { method: 'DELETE' });
const body = await response.json();
if (!response.ok) {
await inspect(id);
throw new Error(body.error ?? body.message ?? 'Could not delete this host.');
}
open = false;
await ondeleted(body);
} catch (cause) { error = (cause as Error).message; }
finally { deleting = false; }
}
</script>
<Dialog bind:open title={`Delete ${host?.name ?? 'host'}?`} busy={deleting}
description="Permanently remove Electric Cherry's files from this host, including its downloaded game files, then uninstall the agent.">
<div class="content">
<p>This deletes the host's game-library copies, server files, agent data, credentials, and service installation. Electric Cherry's uploaded libraries on the controller and its operation history are kept.</p>
<p>The host remains visible until cleanup is confirmed. Offline hosts wait for reconnection. Unsent commands will be cancelled.</p>
{#if checking}<p role="status">Checking servers and unfinished operations…</p>
{:else if deletion && !deletion.allowed}<div role="status" class="blocked"><strong>This host still has work to finish:</strong><ul>{#each deletion.blockers as blocker}<li>{blocker}</li>{/each}</ul><p>Open <a href={`/hosts/${host?.id}`} onclick={() => open = false}>host details</a> to review its operations or drain its servers.</p></div>
{:else if deletion?.allowed}<p>Cleanup cannot be undone once it starts. Adding this machine again requires installing and enrolling the agent again.</p>{/if}
{#if error}<p role="alert" class="ui-error">{error}</p>{/if}
</div>
{#snippet actions()}
<Button variant="ghost" disabled={deleting} onclick={() => open = false}>Cancel</Button>
{#if !deletion?.allowed}<Button disabled={checking || deleting || !host} onclick={() => { if (host) void inspect(host.id); }}>Check again</Button>{/if}
<Button variant="danger" loading={deleting} disabled={checking || !deletion?.allowed} onclick={remove}>Delete host</Button>
{/snippet}
</Dialog>
<style>
.content { display:grid; gap:16px; } p,li { margin:0; color:var(--text-secondary); font-size:var(--text-sm); line-height:1.65; }
.blocked { display:grid; gap:12px; } strong { font-size:var(--text-sm); } ul { margin:0; padding-left:20px; }
a { color:var(--accent-primary-hover); } .ui-error { color:var(--state-danger); }
</style>
@@ -1,6 +1,7 @@
<script lang="ts">
import { onMount } from 'svelte';
import type { Host, NativeDeploymentSelection } from '$lib/hosts.js';
import { hostReadinessLabel } from '$lib/hosts.js';
import type { GameLibrary } from '$lib/game-files.js';
import type { ZombieGame } from '$lib/map-aliases.js';
let { games, selection = $bindable(), ready = $bindable(false), disabled = false, network = false, onchange }:
@@ -26,11 +27,11 @@
<div class="title"><h3>Host and game files</h3><button type="button" disabled={disabled || loading} onclick={() => { onchange?.(); void load(); }}>Refresh hosts</button></div>
{#if error}<p role="alert" class="error">{error}</p>{/if}
{#if loading}<p role="status">Loading host readiness and libraries…</p>{/if}
<label>Game host<select bind:value={selection.hostId} disabled={disabled || loading} onchange={() => onchange?.()}><option value="">Choose a host</option>{#each hosts as candidate}<option value={candidate.id} disabled={candidate.readiness !== 'ready'}>{candidate.name} · {candidate.connectionState} · {candidate.readiness}</option>{/each}</select></label>
<label>Game host<select bind:value={selection.hostId} disabled={disabled || loading} onchange={() => onchange?.()}><option value="">Choose a host</option>{#each hosts as candidate}<option value={candidate.id} disabled={candidate.readiness !== 'ready'}>{candidate.name} · {candidate.connectionState} · {hostReadinessLabel(candidate.readiness)}</option>{/each}</select></label>
{#if !loading && !hosts.some(candidate => candidate.readiness === 'ready')}<p>No deployment-ready hosts. <a href="/hosts" target="_blank" rel="noopener">Set up a host</a>, then refresh.</p>{/if}
{#if host}<p>{host.hostname} · ports {host.portMin}–{host.portMax} · {host.publicAddress || host.bindAddress}</p>{/if}
{#each games as game}<label>{game === 't5' ? 'Black Ops I' : 'Black Ops II'} game-file version<select bind:value={selection.libraryIds[game]} disabled={disabled || loading || !host} onchange={() => onchange?.()}><option value="">Choose an immutable library version</option>{#each versions.filter(version => version.game === game) as version}<option value={version.id} disabled={!cacheReady(version.id)}>{version.name} · {version.version} · {cacheReady(version.id) ? 'Ready on host' : 'Not cached on host'}</option>{/each}</select></label>{/each}
<p><a href="/game-files" target="_blank" rel="noopener">Import or download game files to a host</a>, then refresh. Each launch stays pinned to its selected library version.</p>
<p><a href="/game-files" target="_blank" rel="noopener">Upload a game folder or download a version to a host</a>, then refresh. Each launch stays pinned to its selected library version.</p>
<label>Plutainer runtime image<input required bind:value={selection.image} disabled={disabled} onchange={() => onchange?.()} /></label>
{#if host && !imageReady}<p class="error">Prepare this runtime image from <a href={`/hosts/${host.id}`} target="_blank" rel="noopener">host setup</a> before launch.</p>{/if}
{#if network}<details><summary>Network overrides</summary><label>Local bind address<input bind:value={selection.bindAddress} placeholder={host?.bindAddress ?? 'Host default'} disabled={disabled} onchange={() => onchange?.()} /></label><label>Public / NAT address<input bind:value={selection.publicAddress} placeholder={host?.publicAddress || 'Host default'} disabled={disabled} onchange={() => onchange?.()} /></label><label>Public / NAT port<input type="number" min="1" max="65535" bind:value={selection.publicPort} placeholder="Same as local port" disabled={disabled} onchange={() => onchange?.()} /></label><p>Public mapping does not change the local Docker port reservation.</p></details>{/if}
+38
View File
@@ -0,0 +1,38 @@
export interface SelectedGameFile { path: string; file: File }
export interface SelectedGameFolder { name: string; files: SelectedGameFile[]; size: number }
/** Directory inputs include the selected folder in every relative path. Remove
* that one component and retain the complete game directory below it. */
export function selectGameFolder(selection: Iterable<File>): SelectedGameFolder {
const files: SelectedGameFile[] = [];
const paths = new Set<string>();
let name = '';
let size = 0;
for (const file of selection) {
const [root, ...parts] = file.webkitRelativePath.split('/');
if (!root || root === '.' || root === '..' || !parts.length || name && root !== name) throw new Error('Choose one game folder with the folder picker.');
name = root;
const path = parts.join('/');
if (path.length > 1024 || /[\\\x00-\x1f\x7f:]/.test(path) || parts.some(part => !part || part === '.' || part === '..' || /[. ]$/.test(part))) {
throw new Error(`This file has an unsupported game path: ${file.webkitRelativePath}`);
}
const key = path.toLowerCase();
if (paths.has(key)) throw new Error(`This folder has duplicate game paths: ${path}`);
paths.add(key);
size += file.size;
if (!Number.isSafeInteger(size)) throw new Error('This folder exceeds the supported upload size.');
files.push({ path, file });
if (files.length > 200_000) throw new Error('Choose a folder with no more than 200,000 files.');
}
if (!files.length) throw new Error('The selected folder contains no files.');
for (const { path } of files) {
const parents = path.toLowerCase().split('/');
parents.pop();
while (parents.length) {
if (paths.has(parents.join('/'))) throw new Error(`This folder uses the same game path for a file and a directory: ${path}`);
parents.pop();
}
}
files.sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
return { name, files, size };
}
@@ -0,0 +1,14 @@
import { json } from '@sveltejs/kit';
import { HostDeletionError, receiveDecommissionReceipt } from '$lib/server/hosts/index.js';
import { objectBody, requireAgentTransport } from '$lib/server/hosts/http.js';
/** The uninstall worker retains only this narrowly scoped credential after removing the agent. */
export async function POST({ request, params }: { request: Request; params: { id: string } }) {
requireAgentTransport(request);
const token = request.headers.get('authorization')?.match(/^Bearer ([A-Za-z0-9_-]+)$/i)?.[1] ?? '';
try { return json(receiveDecommissionReceipt(params.id, token, await objectBody(request, 8192)), { headers: { 'Cache-Control': 'no-store' } }); }
catch (cause) {
if (cause instanceof HostDeletionError) return json({ error: cause.message }, { status: cause.status });
throw cause;
}
}
+2 -2
View File
@@ -1,10 +1,10 @@
import { json } from '@sveltejs/kit';
import { createLibraryImport, libraryOverview } from '$lib/server/game-files/index.js';
import { createLibraryUpload, libraryOverview } from '$lib/server/game-files/index.js';
import { gameFileError, readGameJson } from '$lib/server/game-files/http.js';
import { requireManagementOrigin } from '$lib/server/hosts/http.js';
export function GET() { return json(libraryOverview()); }
export async function POST({ request }: { request: Request }) {
try { requireManagementOrigin(request); return json({ version: createLibraryImport(await readGameJson(request)) }, { status: 202 }); }
try { requireManagementOrigin(request); return json({ version: createLibraryUpload(await readGameJson(request, 32 * 1024 * 1024)) }, { status: 202 }); }
catch (cause) { return gameFileError(cause); }
}
@@ -0,0 +1,8 @@
import { json } from '@sveltejs/kit';
import { browserUploadStatus } from '$lib/server/game-files/index.js';
import { gameFileError } from '$lib/server/game-files/http.js';
export async function GET({ params }: { params: { id: string } }) {
try { return json(await browserUploadStatus(params.id), { headers: { 'Cache-Control': 'no-store' } }); }
catch (cause) { return gameFileError(cause); }
}
@@ -0,0 +1,11 @@
import { json } from '@sveltejs/kit';
import { requestBrowserPublication } from '$lib/server/game-files/index.js';
import { gameFileError } from '$lib/server/game-files/http.js';
import { requireManagementOrigin } from '$lib/server/hosts/http.js';
export async function POST({ request, params }: { request: Request; params: { id: string } }) {
try {
requireManagementOrigin(request);
return json(await requestBrowserPublication(params.id), { status: 202 });
} catch (cause) { return gameFileError(cause); }
}
@@ -0,0 +1,17 @@
import { json } from '@sveltejs/kit';
import { uploadBrowserFile } from '$lib/server/game-files/index.js';
import { gameFileError } from '$lib/server/game-files/http.js';
import { requireManagementOrigin } from '$lib/server/hosts/http.js';
function integerHeader(request: Request, name: string): number {
const value = request.headers.get(name);
if (value === null || !/^\d+$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error(`Provide a valid ${name} header.`);
return Number(value);
}
export async function PUT({ request, params }: { request: Request; params: { id: string; path: string } }) {
try {
requireManagementOrigin(request);
const chunk = { offset: integerHeader(request, 'X-Upload-Offset'), size: integerHeader(request, 'X-Upload-Length'), sha256: request.headers.get('X-Content-SHA256') ?? '' };
return json(await uploadBrowserFile(params.id, params.path, chunk, request.body));
} catch (cause) { return gameFileError(cause); }
}
+11 -2
View File
@@ -1,14 +1,23 @@
import { error, json } from '@sveltejs/kit';
import { getHost, listHostOperations, updateHost } from '$lib/server/hosts/index.js';
import { deleteHost, getHost, hostDeletionStatus, HostDeletionError, listHostOperations, updateHost } from '$lib/server/hosts/index.js';
import { hostApiError, objectBody, requireManagementOrigin } from '$lib/server/hosts/http.js';
type Event = { request: Request; params: { id: string } };
export function GET({ request, params }: Event) {
requireManagementOrigin(request); const host = getHost(params.id); if (!host) error(404, 'Host not found.');
// Payloads and results can contain RCON keys or staged credentials. History
// exposes metadata only; agent APIs remain the only command delivery path.
return json({ host, operations: listHostOperations(host.id).map(({ payload: _payload, result: _result, ...operation }) => operation) });
return json({ host, deletion: hostDeletionStatus(host.id), operations: listHostOperations(host.id).map(({ payload: _payload, result: _result, ...operation }) => operation) });
}
export async function PATCH({ request, params }: Event) {
requireManagementOrigin(request);
try { return json({ host: updateHost(params.id, await objectBody(request)) }); } catch (cause) { return hostApiError(cause); }
}
export function DELETE({ request, params }: Event) {
requireManagementOrigin(request);
try { const result = deleteHost(params.id); return json(result, { status: result.deleted ? 200 : 202 }); }
catch (cause) {
if (cause instanceof HostDeletionError) return json({ error: cause.message, blockers: cause.blockers }, { status: cause.status });
return hostApiError(cause);
}
}
+224 -67
View File
@@ -1,9 +1,11 @@
<script lang="ts">
import { onMount } from 'svelte';
import { onMount, tick } from 'svelte';
import { page } from '$app/stores';
import { beforeNavigate } from '$app/navigation';
import type { GameLibrary, GameLibraryVersion } from '$lib/game-files.js';
import { formatGameBytes } from '$lib/game-files.js';
import type { Host } from '$lib/hosts.js';
import { selectGameFolder, type SelectedGameFolder } from '$ui/game-folder.js';
import { pollVisible } from '$ui/poll.js';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Button from '$ui/components/primitives/Button.svelte';
@@ -11,110 +13,265 @@
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
type OverviewLibrary = Omit<GameLibrary, 'versions'> & { versions: Omit<GameLibraryVersion, 'files'>[] };
let libraries = $state<OverviewLibrary[]>([]);
let hosts = $state<Host[]>([]);
let loaded = $state(false);
let error = $state(''); let notice = $state(''); let busy = $state(false);
let showImport = $state(false); let game = $state<'t5' | 't6'>('t5');
let name = $state(''); let version = $state(''); let description = $state(''); let dlc = $state(''); let hostId = $state('');
let filterGame = $state('all');
let targets = $state<Record<string, string>>({});
interface UploadFile { path: string; size: number; lastModified: number; receivedBytes: number; complete: boolean }
interface UploadStatus { status: 'pending' | 'verifying' | 'published' | 'failed'; chunkSize: number; files: UploadFile[]; error?: string | null; version: { id: string; state: string } }
type Phase = 'idle' | 'preparing' | 'uploading' | 'paused' | 'verifying' | 'published';
let libraries = $state<OverviewLibrary[]>([]); let hosts = $state<Host[]>([]);
let loaded = $state(false); let error = $state(''); let notice = $state(''); let busy = $state(false);
let showUpload = $state(false); let game = $state<'t5' | 't6'>('t5');
let name = $state(''); let version = $state(''); let description = $state('');
let filterGame = $state('all'); let targets = $state<Record<string, string>>({});
let folder = $state.raw<SelectedGameFolder | null>(null);
let uploadStatus = $state.raw<UploadStatus | null>(null);
let uploadId = $state(''); let uploadBusy = $state(false); let uploadError = $state('');
let phase = $state<Phase>('idle'); let currentPath = $state(''); let pauseRequested = $state(false);
let acceptedBytes = $state(0); let sendingBytes = $state(0); let completedFiles = $state(0);
let folderInput = $state<HTMLInputElement>();
let activeRequest: XMLHttpRequest | null = null; let disposed = false;
const canManage = $derived($page.data.user?.role === 'admin');
const onlineHosts = $derived(hosts.filter(host => host.connected && host.credentialActive && host.mode !== 'disabled'));
const matching = $derived(libraries.filter(library => filterGame === 'all' || library.game === filterGame));
const hostName = (id: string) => hosts.find(host => host.id === id)?.name ?? id;
const totalBytes = $derived(folder?.size ?? uploadStatus?.files.reduce((total, file) => total + file.size, 0) ?? 0);
const totalFiles = $derived(folder?.files.length ?? uploadStatus?.files.length ?? 0);
const progressBytes = $derived(Math.min(totalBytes, acceptedBytes + sendingBytes));
const transferring = $derived(uploadBusy && (phase === 'preparing' || phase === 'uploading'));
const hostName = (id: string | null) => id ? hosts.find(host => host.id === id)?.name ?? id : 'Browser upload';
async function responseBody(response: Response) {
let body;
try { body = await response.json(); } catch { throw new Error('The controller returned an unexpected response. Sign in again if your session expired, then resume the upload.'); }
if (!response.ok) throw new Error(body.error ?? 'The controller could not complete this request.');
return body;
}
async function load() {
try {
const response = await fetch('/api/game-files');
const body = await response.json();
if (!response.ok) throw new Error(body.error ?? 'Could not load game libraries.');
const body = await responseBody(await fetch('/api/game-files'));
libraries = body.libraries; hosts = body.hosts; loaded = true;
} catch (cause) { error = (cause as Error).message; }
}
async function act(url: string, payload: Record<string, unknown>, message: string) {
async function download(versionId: string) {
busy = true; error = ''; notice = '';
try {
const response = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) });
const result = await response.json();
if (!response.ok) throw new Error(result.error ?? 'Could not start this operation.');
notice = message; await load(); return true;
} catch (cause) { error = (cause as Error).message; return false; }
finally { busy = false; }
await responseBody(await fetch(`/api/game-files/${versionId}/download`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ hostId: targets[versionId] }) }));
notice = 'Library download queued. Progress will update below.'; await load();
} catch (cause) { error = (cause as Error).message; } finally { busy = false; }
}
async function startImport(event: SubmitEvent) {
event.preventDefault();
if (await act('/api/game-files', { game, name, version, description, dlc: dlc.split(',').map(value => value.trim()).filter(Boolean), hostId }, 'Import queued. The source host will scan and upload files, then verify and publish this version.')) showImport = false;
function resetUpload() {
folder = null; uploadStatus = null; uploadId = ''; phase = 'idle'; uploadError = ''; currentPath = '';
acceptedBytes = 0; sendingBytes = 0; completedFiles = 0; pauseRequested = false;
if (folderInput) folderInput.value = '';
}
function newVersion(library: OverviewLibrary) {
showImport = true; game = library.game; name = library.name; description = library.description;
dlc = library.versions[0]?.dlc.join(', ') ?? ''; version = '';
document.getElementById('library-import')?.scrollIntoView({ behavior: 'smooth' });
async function newUpload(library?: OverviewLibrary) {
resetUpload(); showUpload = true; game = library?.game ?? 't5'; name = library?.name ?? '';
description = library?.description ?? ''; version = '';
await tick(); document.getElementById('library-upload')?.scrollIntoView({ behavior: 'smooth' });
}
onMount(() => pollVisible(load, 5000));
function chooseFolder(event: Event) {
uploadError = ''; folder = null;
const input = event.currentTarget as HTMLInputElement;
if (!input.files?.length) return;
try {
const selected = selectGameFolder(Array.from(input.files));
if (uploadStatus) checkSelection(selected, uploadStatus);
folder = selected;
if (!name && !uploadId) name = game === 't5' ? 'Black Ops I game files' : 'Black Ops II game files';
} catch (cause) { uploadError = (cause as Error).message; input.value = ''; }
}
function checkSelection(selected: SelectedGameFolder, status: UploadStatus) {
const originals = new Map(status.files.map(file => [file.path, file]));
if (selected.files.length !== originals.size || selected.files.some(({ path, file }) => {
const original = originals.get(path); return !original || original.size !== file.size || original.lastModified !== file.lastModified;
})) throw new Error('This folder does not match the original upload. Select the same unchanged folder, including its original file names, sizes, and modification dates. Upload changed game files as a new version.');
}
function applyStatus(status: UploadStatus) {
uploadStatus = status;
acceptedBytes = status.files.reduce((total, file) => total + file.receivedBytes, 0);
completedFiles = status.files.filter(file => file.complete).length; sendingBytes = 0;
}
async function getStatus(): Promise<UploadStatus> {
const status = await responseBody(await fetch(`/api/game-files/${uploadId}/upload`)) as UploadStatus;
applyStatus(status); return status;
}
async function followVerification(initial: UploadStatus) {
let status = initial; phase = 'verifying'; currentPath = '';
while (!disposed && status.status === 'verifying') {
await new Promise(resolve => setTimeout(resolve, 1500));
if (disposed) return;
status = await getStatus();
}
if (disposed) return;
if (status.status !== 'published') throw new Error(status.error || 'Verification did not finish. Retry this upload to continue.');
phase = 'published'; folder = null; notice = `${name || 'Game files'} ${version} published. You can download this version to a host below.`;
if (folderInput) folderInput.value = '';
await load();
}
async function resumeUpload(library: OverviewLibrary, item: OverviewLibrary['versions'][number]) {
resetUpload(); showUpload = true; uploadId = item.id; game = library.game; name = library.name; version = item.version; description = item.description;
uploadBusy = true;
try {
const status = await getStatus();
if (status.status === 'verifying' || status.status === 'published') await followVerification(status);
else { phase = 'paused'; uploadError = status.error ?? ''; }
} catch (cause) { uploadError = (cause as Error).message; phase = 'paused'; }
finally { uploadBusy = false; }
await tick(); document.getElementById('library-upload')?.scrollIntoView({ behavior: 'smooth' });
}
function pauseUpload() { pauseRequested = true; activeRequest?.abort(); }
async function sendChunk(path: string, file: File, offset: number, length: number): Promise<number> {
const chunk = file.slice(offset, offset + length);
const hash = await crypto.subtle.digest('SHA-256', await chunk.arrayBuffer());
if (pauseRequested || disposed) throw new Error('Upload paused.');
const sha256 = [...new Uint8Array(hash)].map(value => value.toString(16).padStart(2, '0')).join('');
return new Promise((resolve, reject) => {
const request = new XMLHttpRequest(); activeRequest = request;
request.open('PUT', `/api/game-files/${uploadId}/upload/files/${path.split('/').map(encodeURIComponent).join('/')}`);
request.setRequestHeader('Content-Type', 'application/octet-stream');
request.setRequestHeader('X-Upload-Offset', String(offset)); request.setRequestHeader('X-Upload-Length', String(length));
request.setRequestHeader('X-Content-SHA256', sha256); request.timeout = 300_000;
request.upload.onprogress = event => { sendingBytes = Math.min(length, event.loaded); };
request.onload = () => {
activeRequest = null;
try {
const body = JSON.parse(request.responseText);
if (request.status < 200 || request.status >= 300) throw new Error(body.error ?? `Upload failed (${request.status}).`);
if (body.receivedBytes !== offset + length) throw new Error('The upload position changed. Resume to check the saved progress.');
resolve(body.receivedBytes);
} catch (cause) { reject(cause); }
};
request.onerror = () => { activeRequest = null; reject(new Error('The connection was interrupted. Resume to continue from the last saved chunk.')); };
request.ontimeout = () => { activeRequest = null; reject(new Error('This chunk timed out. Resume to check saved progress and retry.')); };
request.onabort = () => { activeRequest = null; reject(new Error('Upload paused.')); };
request.send(chunk);
});
}
async function startUpload(event: SubmitEvent) {
event.preventDefault(); if (uploadBusy || !folder) return;
if (!globalThis.crypto?.subtle) { uploadError = 'Folder uploads require a secure connection. Open this controller over HTTPS or localhost.'; return; }
uploadBusy = true; pauseRequested = false; uploadError = ''; notice = ''; phase = 'preparing';
try {
if (!uploadId) {
const result = await responseBody(await fetch('/api/game-files', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ game, name, version, description, files: folder.files.map(({ path, file }) => ({ path, size: file.size, lastModified: file.lastModified })) }) }));
uploadId = result.version.id;
}
const status = await getStatus();
if (status.status === 'verifying' || status.status === 'published') { await followVerification(status); return; }
checkSelection(folder, status);
const localFiles = new Map(folder.files.map(item => [item.path, item.file]));
const chunkSize = Math.min(8_388_608, Math.max(1, status.chunkSize));
phase = 'uploading';
for (const remote of status.files) {
if (pauseRequested || disposed) break;
if (remote.complete) continue;
currentPath = remote.path;
const file = localFiles.get(remote.path)!;
do {
if (pauseRequested || disposed) break;
const length = Math.min(chunkSize, remote.size - remote.receivedBytes);
remote.receivedBytes = await sendChunk(remote.path, file, remote.receivedBytes, length);
acceptedBytes += length; sendingBytes = 0;
if (remote.receivedBytes === remote.size) { remote.complete = true; completedFiles++; }
} while (!remote.complete);
}
if (pauseRequested || disposed) { phase = 'paused'; return; }
phase = 'verifying'; currentPath = '';
const verification = await responseBody(await fetch(`/api/game-files/${uploadId}/upload/complete`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' })) as UploadStatus;
applyStatus(verification); await followVerification(verification);
} catch (cause) {
if (!disposed) { uploadError = pauseRequested ? '' : (cause as Error).message; phase = 'paused'; }
} finally { uploadBusy = false; sendingBytes = 0; if (!disposed) void load(); }
}
beforeNavigate(navigation => {
if (!transferring) return;
if (navigation.type === 'leave' || !window.confirm('This upload is still transferring files. Leave this page? You can resume later by selecting the original unchanged folder.')) navigation.cancel();
});
onMount(() => {
const stop = pollVisible(load, 5000);
return () => { disposed = true; pauseRequested = true; activeRequest?.abort(); stop(); };
});
</script>
<svelte:head><title>Game Files | Electric Cherry</title></svelte:head>
<div class="page">
<SectionHeader title="Game Files" subtitle="Import immutable base-game versions and distribute verified copies to your hosts.">
{#snippet actions()}{#if canManage}<Button onclick={() => showImport = !showImport}>{showImport ? 'Close import' : 'Import game files'}</Button>{/if}{/snippet}
<SectionHeader title="Game Files" subtitle="Upload your T5 or T6 game folder, then distribute verified versions to your hosts.">
{#snippet actions()}{#if canManage}<Button disabled={uploadBusy} onclick={() => newUpload()}>Upload game folder</Button>{/if}{/snippet}
</SectionHeader>
{#if error}<p class="ui-error" role="alert">{error}</p>{/if}
{#if notice}<p role="status">{notice}</p>{/if}
{#if !canManage}<p class="muted">Administrators import game files and distribute verified copies to hosts.</p>{/if}
{#if canManage && showImport}
<section class="card" id="library-import">
<h2>Import an existing game volume</h2>
<p>The selected host scans its {game}_game_files Docker volume, calculates SHA256 checksums, and streams a copy into controller storage. Publication requires every file to pass verification.</p>
<form onsubmit={startImport}>
<div class="fields">
{#if !canManage}<p class="muted">Administrators upload game files and distribute verified copies to hosts.</p>{/if}
{#if canManage && showUpload}
<section class="card" id="library-upload">
<div class="heading"><h2>{uploadId ? phase === 'published' ? 'Version published' : 'Continue folder upload' : 'Upload a game folder'}</h2><Button size="sm" variant="ghost" disabled={uploadBusy} onclick={() => showUpload = false}>Close</Button></div>
<p>Select your complete T5 or T6 game folder, including its installed DLC. The files upload directly to this controller; a game host is only needed when you are ready to run servers.</p>
{#if uploadError}<p class="ui-error" role="alert">{uploadError}</p>{/if}
<form onsubmit={startUpload}>
<fieldset disabled={uploadBusy || !!uploadId} class="fields">
<label>Game<select bind:value={game}><option value="t5">Black Ops I (T5)</option><option value="t6">Black Ops II (T6)</option></select></label>
<label>Source host<select required bind:value={hostId}><option value="">Select an online host</option>{#each onlineHosts as host}<option value={host.id}>{host.name}</option>{/each}</select></label>
<label>Library name<input required maxlength="160" bind:value={name} placeholder="Black Ops II — Complete Server Files" /></label>
<label>New version label<input required maxlength="120" bind:value={version} placeholder="2026-10-03" /></label>
<label class="wide">Description<textarea maxlength="4000" bind:value={description} rows="2"></textarea></label>
<label class="wide">Content / DLC (comma separated)<input bind:value={dlc} placeholder="Base game, Nuketown, Die Rise, Mob of the Dead, Buried, Origins" /></label>
<label>Library name<input required maxlength="160" bind:value={name} placeholder="Black Ops II game files" /></label>
<label>Version label<input required maxlength="120" bind:value={version} placeholder="2026-10-04" /></label>
<label class="wide">Description <span class="muted">(optional)</span><textarea maxlength="4000" bind:value={description} rows="2"></textarea></label>
</fieldset>
{#if phase !== 'verifying' && phase !== 'published'}
<label class="folder-picker">{uploadId ? 'Select the original unchanged folder' : 'Game folder'}<input bind:this={folderInput} type="file" webkitdirectory multiple onchange={chooseFolder} disabled={uploadBusy} aria-describedby="folder-help" /></label>
<p class="muted" id="folder-help">{uploadId ? 'To resume, file names, sizes, and modification dates must match the original upload.' : 'Choose the game directory itself. Its top-level folder name is removed once; all files and subfolders inside it keep their paths.'}</p>
{/if}
{#if folder}<div class="folder-summary"><strong>{folder.name}</strong><span>{folder.files.length.toLocaleString()} files · {formatGameBytes(folder.size)}</span><details><summary>Preview folder paths</summary><ul class="path-list">{#each folder.files.slice(0, 8) as file}<li>{file.path}</li>{/each}</ul>{#if folder.files.length > 8}<p class="muted">And {(folder.files.length - 8).toLocaleString()} more files.</p>{/if}</details></div>{/if}
{#if phase !== 'idle'}<div class="upload-progress" aria-live="polite">
<div class="heading"><strong>{phase === 'preparing' ? 'Preparing upload…' : phase === 'uploading' ? 'Uploading game files' : phase === 'verifying' ? 'Verifying and publishing…' : phase === 'published' ? 'Published and ready to distribute' : 'Upload paused'}</strong><span>{completedFiles.toLocaleString()} / {totalFiles.toLocaleString()} files</span></div>
<progress max={totalBytes || Math.max(1, totalFiles)} value={totalBytes ? progressBytes : completedFiles} aria-label="Game folder upload progress"></progress>
<p>{formatGameBytes(progressBytes)} / {formatGameBytes(totalBytes)} saved or transferring</p>
{#if currentPath}<p class="current-file">{currentPath}</p>{/if}
{#if phase === 'verifying'}<p class="muted">The controller checks every file and publishes this version when verification finishes. Verification continues if you leave this page.</p>{:else if phase === 'paused'}<p class="muted">Saved chunks are kept on the controller. Resume this upload to check the saved progress and retry.</p>{/if}
</div>{/if}
<div class="upload-actions">
{#if transferring}<Button onclick={pauseUpload} disabled={pauseRequested}>{pauseRequested ? 'Pausing…' : 'Pause upload'}</Button>
{:else if phase !== 'verifying' && phase !== 'published'}<Button type="submit" variant="primary" disabled={uploadBusy || !folder}>{uploadId ? 'Resume upload' : 'Upload and publish'}</Button>{/if}
{#if phase === 'idle' || phase === 'uploading'}<span class="muted">Keep this page open while files transfer. Interrupted uploads can be resumed below.</span>{/if}
</div>
<Button type="submit" disabled={busy || !onlineHosts.length}>Start import</Button>
{#if !onlineHosts.length}<p>Enroll an online source host in <a href="/hosts">Hosts</a> before importing.</p>{/if}
</form>
</section>
{/if}
<div class="filters"><label>Game<select bind:value={filterGame}><option value="all">All games</option><option value="t5">Black Ops I</option><option value="t6">Black Ops II</option></select></label><span class="muted">{libraries.length} libraries · Base-game files and runtime images are versioned separately.</span></div>
<div class="filters"><label>Game<select bind:value={filterGame}><option value="all">All games</option><option value="t5">Black Ops I</option><option value="t6">Black Ops II</option></select></label><span class="muted">{libraries.length} libraries · Game folders and runtime images are versioned separately.</span></div>
{#each matching as library (library.id)}
<section class="card">
<div class="heading"><div><h2>{library.name}</h2><p class="muted">{library.game.toUpperCase()} · {library.description}</p></div>{#if canManage}<Button size="sm" onclick={() => newVersion(library)}>Import new version</Button>{/if}</div>
<div class="heading"><div><h2>{library.name}</h2><p class="muted">{library.game.toUpperCase()} · {library.description}</p></div>{#if canManage}<Button size="sm" disabled={uploadBusy} onclick={() => newUpload(library)}>Upload new version</Button>{/if}</div>
{#each library.versions as item (item.id)}
{@const browserUpload = !item.sourceHostId}
<article class="version">
<div class="heading"><h3>{item.version}</h3><Badge variant={item.state === 'failed' ? 'danger' : item.state === 'ready' ? 'online' : 'info'}>{item.state === 'ready' ? 'Published' : item.state === 'importing' ? 'Importing' : 'Failed'}</Badge></div>
<div class="heading"><h3>{item.version}</h3><Badge variant={item.state === 'failed' ? 'danger' : item.state === 'ready' ? 'online' : 'info'}>{item.state === 'ready' ? 'Published' : item.state === 'importing' ? browserUpload ? item.id === uploadId && phase === 'verifying' ? 'Verifying' : 'Uploading' : 'Importing' : 'Failed'}</Badge></div>
<p>{formatGameBytes(item.sizeBytes)} · {item.fileCount.toLocaleString()} files · Created {new Date(item.createdAt).toLocaleString()}</p>
{#if item.dlc.length}<p>Content: {item.dlc.join(', ')}</p>{/if}
{#if item.error}<p class="ui-error">{item.error}</p>{/if}
<details><summary>Version details</summary><dl><dt>Immutable version ID</dt><dd>{item.id}</dd><dt>Manifest SHA256</dt><dd>{item.manifestSha256 ?? 'Waiting for source scan'}</dd><dt>Controller storage</dt><dd>{item.storageLocation}</dd><dt>Imported from</dt><dd>{hostName(item.sourceHostId)} / {item.sourceVolume}</dd></dl><a href={`/api/game-files/${item.id}`} target="_blank" rel="noreferrer">View complete file manifest</a></details>
<details><summary>Version details</summary><dl><dt>Immutable version ID</dt><dd>{item.id}</dd><dt>Manifest SHA256</dt><dd>{item.manifestSha256 ?? 'Available after verification'}</dd><dt>Uploaded from</dt><dd>{browserUpload ? 'Game folder in browser' : `${hostName(item.sourceHostId)} / ${item.sourceVolume}`}</dd></dl>{#if item.state === 'ready'}<a href={`/api/game-files/${item.id}`} target="_blank" rel="noreferrer">View complete file manifest</a>{/if}</details>
{#if item.state === 'ready'}
{#if canManage}<div class="download"><label>Download to host<select bind:value={targets[item.id]}><option value="">Select host</option>{#each onlineHosts as host}<option value={host.id}>{host.name} · {formatGameBytes(Number(host.inventory.storageAvailable ?? 0))} free</option>{/each}</select></label><Button size="sm" disabled={busy || !targets[item.id]} onclick={() => act(`/api/game-files/${item.id}/download`, { hostId: targets[item.id] }, 'Library download queued. Progress will update below.')}>Download / verify</Button></div>{/if}
{#if canManage}<div class="download"><label>Download to host<select bind:value={targets[item.id]}><option value="">Select host</option>{#each onlineHosts as host}<option value={host.id}>{host.name} · {formatGameBytes(Number(host.inventory.storageAvailable ?? 0))} free</option>{/each}</select></label><Button size="sm" disabled={busy || !targets[item.id]} onclick={() => download(item.id)}>Download / verify</Button></div>{#if !onlineHosts.length}<p class="muted">This version is stored and ready. <a href="/hosts">Connect a host</a> when you want to deploy game servers.</p>{/if}{/if}
<div class="table-wrap"><table><thead><tr><th>Host</th><th>Cache state</th><th>Progress</th><th>Detail</th></tr></thead><tbody>
{#each item.caches as cache}<tr><td><a href={`/hosts/${cache.hostId}`}>{hostName(cache.hostId)}</a></td><td>{cache.status}{!hosts.find(host => host.id === cache.hostId)?.connected ? ' · offline' : ''}</td><td>{#if cache.status === 'downloading' || cache.status === 'verifying'}<progress max={cache.bytesTotal || 1} value={cache.bytesDownloaded}></progress><small>{formatGameBytes(cache.bytesDownloaded)} / {formatGameBytes(cache.bytesTotal)}</small>{:else if cache.status === 'ready' || cache.status === 'outdated'}Verified{cache.status === 'outdated' ? ' · older version' : ''}{:else}—{/if}</td><td>{cache.error ?? ''}</td></tr>{:else}<tr><td colspan="4">Enroll a host to cache this version.</td></tr>{/each}
{#each item.caches as cache}<tr><td><a href={`/hosts/${cache.hostId}`}>{hostName(cache.hostId)}</a></td><td>{cache.status}{!hosts.find(host => host.id === cache.hostId)?.connected ? ' · offline' : ''}</td><td>{#if cache.status === 'downloading' || cache.status === 'verifying'}<progress max={cache.bytesTotal || 1} value={cache.bytesDownloaded} aria-label="Host download progress"></progress><small>{formatGameBytes(cache.bytesDownloaded)} / {formatGameBytes(cache.bytesTotal)}</small>{:else if cache.status === 'ready' || cache.status === 'outdated'}Verified{cache.status === 'outdated' ? ' · older version' : ''}{:else}—{/if}</td><td>{cache.error ?? ''}</td></tr>{:else}<tr><td colspan="4">No host copies yet.</td></tr>{/each}
</tbody></table></div>
{:else if item.state === 'importing'}<p class="muted">The agent operation continues independently of this page. An interrupted import is reconciled when the host reconnects.</p>{/if}
{:else if browserUpload}<p class="muted">Resume with the original unchanged folder. Published versions can be downloaded to any enrolled host.</p>{#if canManage}<div><Button size="sm" disabled={uploadBusy} onclick={() => resumeUpload(library, item)}>Resume upload</Button></div>{/if}
{:else if item.state === 'importing'}<p class="muted">The existing host import continues independently. It is reconciled when the host reconnects.</p>{/if}
</article>
{/each}
</section>
{:else}{#if loaded}<EmptyState title="No game libraries yet" description="Administrators can import T5 or T6 game files from an enrolled host. Each published version can be shared by many instances." />{:else}<p role="status">Loading game libraries…</p>{/if}{/each}
{:else}{#if loaded}<EmptyState title="No game libraries yet" description="Upload a complete T5 or T6 game folder to get started. You can add a host after the files are published." />{:else}<p role="status">Loading game libraries…</p>{/if}{/each}
</div>
<style>
.page, .card, .version, form { display: grid; gap: 20px; }
.card { padding: 24px; border: 1px solid var(--border-subtle); background: var(--surface); border-radius: var(--radius-md); }
.version { padding-top: 20px; border-top: 1px solid var(--border-subtle); }
.heading, .filters, .download { display: flex; gap: 16px; align-items: center; justify-content: space-between; flex-wrap: wrap; }
.download { justify-content: flex-start; align-items: end; }
.fields { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 16px; }
.wide { grid-column: 1 / -1; } label { display: grid; gap: 6px; font-size: var(--text-sm); }
input, select, textarea { width: 100%; min-width: 0; box-sizing: border-box; padding: 10px; background: var(--surface); color: inherit; border: 1px solid var(--border-strong); border-radius: var(--radius-sm); }
h2, h3, p { margin: 0; } h2 { font-size: var(--text-md); } h3 { font-size: var(--text-sm); }
p, td, th, details { font-size: var(--text-sm); line-height: 1.6; } .muted, small { color: var(--text-secondary); }
small { display: block; } a { color: var(--accent-primary-hover); } summary { cursor: pointer; }
dl { display: grid; grid-template-columns: auto 1fr; gap: 6px 16px; } dd { margin: 0; overflow-wrap: anywhere; }
.table-wrap { overflow-x: auto; } table { width: 100%; border-collapse: collapse; text-align: left; } th, td { padding: 10px; border-bottom: 1px solid var(--border-subtle); }
@media(max-width: 640px) { .fields { grid-template-columns: 1fr; } .card { padding: 16px; } dl { grid-template-columns: 1fr; } }
.page, .card, .version, form { display:grid; gap:20px; }
.card { padding:24px; min-width:0; border:1px solid var(--border-subtle); background:var(--surface); border-radius:var(--radius-md); }
.version { padding-top:20px; border-top:1px solid var(--border-subtle); }
.heading, .filters, .download, .upload-actions { display:flex; gap:16px; align-items:center; justify-content:space-between; flex-wrap:wrap; }
.download, .upload-actions { justify-content:flex-start; align-items:end; }
.fields { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:16px; border:0; padding:0; margin:0; min-width:0; }
.wide { grid-column:1/-1; } label { display:grid; gap:6px; font-size:var(--text-sm); }
input, select, textarea { width:100%; min-width:0; box-sizing:border-box; padding:10px; background:var(--surface); color:inherit; border:1px solid var(--border-strong); border-radius:var(--radius-sm); }
input:disabled, select:disabled, textarea:disabled { opacity:.7; } input[type='file'] { padding:16px; cursor:pointer; }
.folder-summary,.upload-progress { padding:20px; display:grid; gap:12px; background:var(--surface-sunken); border:1px solid var(--border-subtle); border-radius:var(--radius-md); min-width:0; }
.folder-summary>span,.upload-progress span,.upload-actions>span { color:var(--text-secondary); font-size:var(--text-sm); }
.path-list { padding-left:20px; font-family:var(--font-mono); font-size:var(--text-xs); overflow-wrap:anywhere; }
.current-file { font-family:var(--font-mono); font-size:var(--text-xs); overflow-wrap:anywhere; } progress { width:100%; accent-color:var(--accent-primary); }
h2,h3,p { margin:0; } h2 { font-size:var(--text-md); } h3 { font-size:var(--text-sm); }
p,td,th,details { font-size:var(--text-sm); line-height:1.6; } .muted,small { color:var(--text-secondary); }
small { display:block; } a { color:var(--accent-primary-hover); } summary { cursor:pointer; }
dl { display:grid; grid-template-columns:auto 1fr; gap:6px 16px; } dd { margin:0; overflow-wrap:anywhere; }
.table-wrap { overflow-x:auto; } table { width:100%; border-collapse:collapse; text-align:left; } th,td { padding:10px; border-bottom:1px solid var(--border-subtle); }
@media(max-width:640px) { .fields { grid-template-columns:1fr; } .card,.folder-summary,.upload-progress { padding:16px; } dl { grid-template-columns:1fr; } .heading>div { min-width:0; } }
</style>
+17 -4
View File
@@ -2,16 +2,27 @@
import { onMount } from 'svelte';
import { page } from '$app/stores';
import type { Host } from '$lib/hosts.js';
import { hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Button from '$ui/components/primitives/Button.svelte';
import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
let hosts = $state<Host[]>([]);
let loading = $state(true); let busy = $state(false); let error = $state('');
let notice = $state(''); let deleteOpen = $state(false); let deletingHost = $state<Host | null>(null);
let loadSequence = 0;
let name = $state(''); let portMin = $state(28960); let portMax = $state(28980);
let enrollment = $state<{ command: string; expiresAt: string; hostId: string } | null>(null);
const canManage = $derived($page.data.user?.role === 'admin');
async function load() {
try { const response = await fetch('/api/hosts'); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); hosts = body.hosts; }
catch (cause) { error = (cause as Error).message; } finally { loading = false; }
const sequence = ++loadSequence;
try { const response = await fetch('/api/hosts'); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? 'Could not load hosts.'); if (sequence === loadSequence) { hosts = body.hosts; error = ''; } }
catch (cause) { if (sequence === loadSequence) error = (cause as Error).message; } finally { if (sequence === loadSequence) loading = false; }
}
async function deleted(result: { deleted: boolean }) {
notice = result.deleted ? `${deletingHost?.name ?? 'Host'} deleted.` : `Cleanup requested for ${deletingHost?.name ?? 'host'}. It remains listed until its files and agent are removed.`;
if (enrollment?.hostId === deletingHost?.id) enrollment = null;
deletingHost = null;
await load();
}
async function add(event: SubmitEvent) {
event.preventDefault(); busy = true; error = ''; enrollment = null;
@@ -28,6 +39,7 @@
<section class="page">
<SectionHeader title="Hosts" subtitle="Connect game-server hosts and prepare them for deployment." />
{#if error}<p role="alert" class="ui-error">{error}</p>{/if}
{#if notice}<p role="status">{notice}</p>{/if}
{#if canManage}<section class="panel">
<h2>Add host</h2><p>Run the installation command on a Debian 12 or 13 amd64 machine. Each host connects to this controller over HTTPS.</p>
<form onsubmit={add}><fieldset disabled={busy}>
@@ -39,14 +51,15 @@
{#if enrollment}<div class="enrollment" role="status"><h3>Install the agent</h3><p>This single-use command expires {new Date(enrollment.expiresAt).toLocaleString()}. Run it with administrator access on your game host.</p><pre>{enrollment.command}</pre><a href={`/hosts/${enrollment.hostId}`}>Open host setup →</a></div>{/if}
</section>{:else}<p>Administrators manage host setup, runtime preparation, and credentials.</p>{/if}
{#if loading}<p role="status">Loading hosts…</p>{:else if !hosts.length}<p>No hosts connected yet.{canManage ? ' Add a host to begin.' : ''}</p>{:else}
<div class="host-grid">{#each hosts as host}<a class="panel host" href={`/hosts/${host.id}`}><div class="row"><h2>{host.name}</h2><span class:ready={host.readiness === 'ready'} class="state">{host.readiness}</span></div><p>{host.hostname || 'Awaiting enrollment'} · {host.connectionState}</p><p>{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}</p><p>Game ports {host.portMin}–{host.portMax}</p>{#if host.readinessReasons.length}<p class="reason">{host.readinessReasons[0]}</p>{/if}</a>{/each}</div>
<div class="host-grid">{#each hosts as host (host.id)}<article class="panel host"><div class="row"><h2><a href={`/hosts/${host.id}`}>{host.name}</a></h2><span class:ready={host.readiness === 'ready'} class="state">{hostReadinessLabel(host.readiness)}</span></div><p>{host.hostname || 'Awaiting enrollment'} · {host.connectionState}</p><p>{host.inventory.os ?? 'OS pending'} · {host.inventory.architecture ?? 'Architecture pending'}</p><p>Game ports {host.portMin}–{host.portMax}</p>{#if host.decommission}<p class="reason">{hostDeletionMessage(host)}</p>{#if host.decommission.error}<p class="ui-error">{host.decommission.error}</p>{/if}{:else if host.readinessReasons.length}<p class="reason">{host.readinessReasons[0]}</p>{/if}<div class="host-actions"><Button size="sm" href={`/hosts/${host.id}`}>{host.decommission ? 'View cleanup' : 'Manage host'}</Button>{#if host.readiness === 'needs-game-files'}<Button size="sm" href="/game-files">Choose game files</Button>{/if}{#if canManage && (!host.decommission || host.decommission.retryable)}<Button size="sm" variant="danger" onclick={() => { deletingHost = host; deleteOpen = true; }}>{host.decommission ? 'Retry cleanup' : 'Delete host'}</Button>{/if}</div></article>{/each}</div>
{/if}
</section>
<DeleteHostDialog bind:open={deleteOpen} host={deletingHost} ondeleted={deleted} />
<style>
.page { display:grid; gap:24px; } .panel { border:1px solid var(--border-strong); border-radius:var(--radius-md); background:var(--surface); padding:24px; }
h2 { font-size:var(--text-md); margin:0; } h3 { font-size:var(--text-sm); } p { color:var(--text-secondary); line-height:1.6; font-size:var(--text-sm); }
fieldset { display:flex; gap:16px; flex-wrap:wrap; align-items:end; border:0; padding:0; margin-top:20px; } label { display:grid; gap:8px; font-size:var(--text-sm); } input { width:100%; max-width:280px; padding:10px; color:var(--text-primary); background:var(--surface-sunken); border:1px solid var(--border-control); border-radius:var(--radius-sm); }
pre { white-space:pre-wrap; overflow-wrap:anywhere; padding:16px; background:var(--surface-sunken); font-size:var(--text-sm); } .enrollment { border-top:1px solid var(--border-subtle); margin-top:24px; padding-top:16px; }
.host-grid { display:grid; grid-template-columns:repeat(auto-fit,minmax(300px,1fr)); gap:16px; } .host { text-decoration:none; color:inherit; } .host:hover { border-color:var(--accent-primary); } .row { display:flex; justify-content:space-between; gap:16px; } .state { font-size:var(--text-xs); text-transform:capitalize; color:var(--text-secondary); } .ready { color:var(--accent-primary); } .reason { margin-bottom:0; }
.host-grid { display:grid; grid-template-columns:repeat(auto-fit,minmax(300px,1fr)); gap:16px; } .host h2 a { color:inherit; text-decoration:none; } .host h2 a:hover { color:var(--accent-primary); } .host-actions { display:flex; flex-wrap:wrap; gap:12px; margin-top:20px; } .row { display:flex; justify-content:space-between; gap:16px; } .state { font-size:var(--text-xs); text-transform:capitalize; color:var(--text-secondary); } .ready { color:var(--accent-primary); } .reason { margin-bottom:0; }
</style>
+17 -7
View File
@@ -1,19 +1,24 @@
<script lang="ts">
import { onMount } from 'svelte';
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import type { Host, HostOperation, HostMode } from '$lib/hosts.js';
import { hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Button from '$ui/components/primitives/Button.svelte';
import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
let host = $state<Host | null>(null); let operations = $state<HostOperation[]>([]);
let loading = $state(true); let busy = $state(false); let error = $state(''); let notice = $state('');
let deleteOpen = $state(false);
let name = $state(''); let portMin = $state(28960); let portMax = $state(28980); let bindAddress = $state('0.0.0.0'); let publicAddress = $state(''); let minimumFreeGB = $state(1);
let runtimeImage = $state('ghcr.io/ayymoss/plutainer:latest'); let enrollment = $state<{ command: string; expiresAt: string } | null>(null);
const endpoint = $derived(`/api/hosts/${$page.params.id}`);
const canManage = $derived($page.data.user?.role === 'admin');
const cleaning = $derived(!!host?.decommission);
function bytes(value: unknown) { return typeof value === 'number' ? `${(value / 1073741824).toFixed(1)} GiB` : 'Unknown'; }
async function load(initialize = false) {
try {
const response = await fetch(endpoint); const body = await response.json(); if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations;
const response = await fetch(endpoint); const body = await response.json(); if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations;
if (initialize && host) { name = host.name; portMin = host.portMin; portMax = host.portMax; bindAddress = host.bindAddress; publicAddress = host.publicAddress; minimumFreeGB = host.minimumFreeBytes / 1073741824; }
} catch (cause) { error = (cause as Error).message; } finally { loading = false; }
}
@@ -28,27 +33,32 @@
</script>
<svelte:head><title>{host?.name ?? 'Host'} · Electric Cherry</title></svelte:head>
<section class="page"><a href="/hosts">← All hosts</a><SectionHeader title={host?.name ?? 'Host'} subtitle="Readiness, inventory, credentials, and operation history." />
<section class="page"><a href="/hosts">← All hosts</a><SectionHeader title={host?.name ?? 'Host'} subtitle="Readiness, inventory, credentials, and operation history.">
{#snippet actions()}{#if canManage && host && (!cleaning || host.decommission?.retryable)}<Button variant="danger" disabled={busy} onclick={() => deleteOpen = true}>{cleaning ? 'Retry cleanup' : 'Delete host'}</Button>{/if}{/snippet}
</SectionHeader>
{#if error}<p class="ui-error" role="alert">{error}</p>{/if}{#if notice}<p role="status">{notice}</p>{/if}
{#if loading}<p role="status">Loading host…</p>{:else if host}
<section class="panel"><div class="row"><h2>{host.connectionState} · {host.readiness}</h2><span>Last heartbeat: {host.lastHeartbeat ? new Date(host.lastHeartbeat).toLocaleString() : 'Never'}</span></div>
{#if host.decommission}<section class="panel" aria-live="polite"><h2>Host cleanup</h2><p>{hostDeletionMessage(host)}</p>{#if host.decommission.error}<p class="ui-error">{host.decommission.error}</p>{/if}<p>Requested {new Date(host.decommission.requestedAt).toLocaleString()}. This page closes only after the cleanup worker confirms removal.</p>{#if host.decommission.stage === 'cleaned'}<p>The agent is expected to go offline during uninstall. If final confirmation never arrives, inspect the machine; loss of contact alone does not prove completion.</p>{/if}</section>{/if}
<section class="panel"><div class="row"><h2>{host.connectionState} · {hostReadinessLabel(host.readiness)}</h2><span>Last heartbeat: {host.lastHeartbeat ? new Date(host.lastHeartbeat).toLocaleString() : 'Never'}</span></div>
{#if host.readinessReasons.length}<ul>{#each host.readinessReasons as reason}<li>{reason}</li>{/each}</ul>{:else}<p>Connected and ready. Each deployment also checks its game library, runtime, and ports.</p>{/if}
{#if canManage}<div class="actions"><Button disabled={busy || host.mode === 'active'} onclick={() => mode('active')}>Enable</Button><Button disabled={busy || host.mode === 'draining'} onclick={() => mode('draining')}>Drain host</Button><Button disabled={busy || host.mode === 'disabled'} onclick={() => mode('disabled')}>Disable host</Button></div>{/if}
{#if host.readiness === 'needs-game-files'}<p><a href="/game-files">Choose a published game-file version and download it to this host →</a></p>{/if}
{#if canManage && !cleaning}<div class="actions"><Button disabled={busy || host.mode === 'active'} onclick={() => mode('active')}>Enable</Button><Button disabled={busy || host.mode === 'draining'} onclick={() => mode('draining')}>Drain host</Button><Button disabled={busy || host.mode === 'disabled'} onclick={() => mode('disabled')}>Disable host</Button></div>{/if}
<p>Draining prevents new deployments and retires instances as they become safe to remove. Disabled hosts receive no new commands; running game containers continue.</p>
</section>
<div class="grid"><section class="panel"><h2>Inventory</h2><dl>
<dt>Hostname</dt><dd>{host.hostname || 'Pending'}</dd><dt>Agent</dt><dd>{host.inventory.agentVersion ?? 'Unknown'}</dd><dt>Platform</dt><dd>{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}</dd><dt>CPU</dt><dd>{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)</dd><dt>Memory available / total</dt><dd>{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}</dd><dt>Game storage available / total</dt><dd>{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}</dd><dt>Docker</dt><dd>{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}</dd><dt>Current operation</dt><dd>{host.inventory.currentOperation ?? 'None reported'}</dd>
</dl></section><section class="panel"><h2>Host configuration</h2>{#if canManage}<form onsubmit={save}><fieldset disabled={busy}>
</dl></section><section class="panel"><h2>Host configuration</h2>{#if canManage}<form onsubmit={save}><fieldset disabled={busy || cleaning}>
<label>Friendly name<input required maxlength="200" bind:value={name} /></label><div class="fields"><label>First game port<input type="number" min="1" max="65535" required bind:value={portMin} /></label><label>Last game port<input type="number" min={portMin} max="65535" required bind:value={portMax} /></label></div><label>Local bind address<input required bind:value={bindAddress} /></label><label>Public / NAT address<input bind:value={publicAddress} placeholder="Optional public hostname or address" /></label><label>Minimum free game storage (GiB)<input type="number" min="0" step="0.1" required bind:value={minimumFreeGB} /></label><Button type="submit" disabled={busy}>Save settings</Button>
</fieldset></form>{:else}<dl><dt>Friendly name</dt><dd>{host.name}</dd><dt>Game ports</dt><dd>{host.portMin}–{host.portMax}</dd><dt>Local bind address</dt><dd>{host.bindAddress}</dd><dt>Public / NAT address</dt><dd>{host.publicAddress || 'Not configured'}</dd><dt>Minimum free storage</dt><dd>{bytes(host.minimumFreeBytes)}</dd></dl><p>Host configuration is managed by administrators.</p>{/if}<p>Public addressing is separate from local port reservations. Configure your firewall and NAT for the game-port range.</p></section></div>
<div class="grid"><section class="panel"><h2>Game files and runtime</h2><p><a href="/game-files">Manage game-file libraries →</a></p>{#if host.inventory.libraries?.length}<ul>{#each host.inventory.libraries as library}<li>{library.id}: {library.state}{library.progress !== undefined ? ` (${library.progress}%)` : ''}</li>{/each}</ul>{:else}<p>No cached game libraries reported.</p>{/if}
{#if canManage}<form onsubmit={(event) => { event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}><fieldset disabled={busy}><label>Plutainer image<input required bind:value={runtimeImage} /></label><Button type="submit" disabled={busy || !host.connected}>Prepare runtime</Button></fieldset></form>{/if}<p>Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported'}</p>
</section><section class="panel"><h2>Agent access</h2><p>Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.</p>{#if canManage}<div class="actions"><Button disabled={busy} onclick={() => action('/enrollment', 'POST')}>Generate installer</Button><Button disabled={busy || !host.credentialActive || !host.connected} onclick={() => action('/credentials', 'POST', undefined, 'Credential rotation queued.')}>Rotate credential</Button><Button disabled={busy || !host.credentialActive} onclick={() => action('/credentials', 'DELETE', undefined, 'Host credentials revoked. Generate a new installer to enroll again.')}>Revoke credential</Button></div>
{#if canManage}<form onsubmit={(event) => { event.preventDefault(); void action('/operations', 'POST', { kind: 'runtime.pull', image: runtimeImage }, 'Runtime download queued.'); }}><fieldset disabled={busy || cleaning}><label>Plutainer image<input required bind:value={runtimeImage} /></label><Button type="submit" disabled={busy || !host.connected}>Prepare runtime</Button></fieldset></form>{/if}<p>Cached images: {host.inventory.runtimeImages?.join(', ') || 'None reported'}</p>
</section><section class="panel"><h2>Agent access</h2><p>Credentials: {host.credentialActive ? 'Active' : 'Not active'}. Enrollment links expire after 15 minutes and work once.</p>{#if canManage}<div class="actions"><Button disabled={busy || (cleaning && !host.decommission?.retryable)} onclick={() => action('/enrollment', 'POST')}>Generate installer</Button><Button disabled={busy || cleaning || !host.credentialActive || !host.connected} onclick={() => action('/credentials', 'POST', undefined, 'Credential rotation queued.')}>Rotate credential</Button><Button disabled={busy || cleaning || !host.credentialActive} onclick={() => action('/credentials', 'DELETE', undefined, 'Host credentials revoked. Generate a new installer to enroll again.')}>Revoke credential</Button></div>
{#if enrollment}<p>Expires {new Date(enrollment.expiresAt).toLocaleString()}</p><pre>{enrollment.command}</pre>{/if}{:else}<p>Administrators manage enrollment and agent credentials.</p>{/if}
</section></div>
<section class="panel"><h2>Operation history</h2>{#if !operations.length}<p>No remote operations yet.</p>{:else}<div class="table-wrap"><table><thead><tr><th>Operation</th><th>State</th><th>Updated</th><th>Result</th></tr></thead><tbody>{#each operations as operation}<tr><td>{operation.kind}<small>{operation.id}</small></td><td>{operation.state}</td><td>{new Date(operation.updatedAt).toLocaleString()}</td><td>{operation.error ?? '—'}</td></tr>{/each}</tbody></table></div>{/if}</section>
{/if}
</section>
<DeleteHostDialog bind:open={deleteOpen} {host} ondeleted={async (result) => { if (result.deleted) await goto('/hosts'); else { enrollment = null; await load(); } }} />
<style>
.page { display:grid; gap:24px; } .panel { border:1px solid var(--border-strong); border-radius:var(--radius-md); background:var(--surface); padding:24px; min-width:0; } h2 { margin:0 0 16px; font-size:var(--text-md); text-transform:capitalize; } p, li, dd, dt, span { font-size:var(--text-sm); line-height:1.65; } p,li,dt,span { color:var(--text-secondary); } .grid { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:24px; } .row,.actions,.fields { display:flex; gap:12px; flex-wrap:wrap; } .row { justify-content:space-between; } .actions { margin-top:20px; } dl { display:grid; grid-template-columns:1fr 1.3fr; gap:12px; margin:0; } dd { margin:0; overflow-wrap:anywhere; } fieldset { display:grid; gap:16px; border:0; padding:0; margin:0; } label { display:grid; gap:8px; font-size:var(--text-sm); } input { width:100%; min-width:0; padding:10px; color:var(--text-primary); background:var(--surface-sunken); border:1px solid var(--border-control); border-radius:var(--radius-sm); } .fields>* { flex:1; min-width:120px; } pre { white-space:pre-wrap; overflow-wrap:anywhere; padding:16px; background:var(--surface-sunken); font-size:var(--text-xs); } .table-wrap { overflow:auto; } table { width:100%; border-collapse:collapse; text-align:left; font-size:var(--text-sm); } td,th { padding:12px; border-bottom:1px solid var(--border-subtle); } small { display:block; color:var(--text-secondary); font-size:var(--text-xs); margin-top:4px; } @media(max-width:900px) { .grid { grid-template-columns:1fr; } }
+338 -4
View File
@@ -29,10 +29,17 @@ import urllib.parse
import urllib.request
import uuid
VERSION = "1.0.0"
VERSION = "1.1.0"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
AGENT_CONFIG = Path("/etc/electric-cherry/agent.json")
AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py")
AGENT_UNIT = Path("/etc/systemd/system/electric-cherry-agent.service")
UNINSTALL_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-uninstall.py")
UNINSTALL_UNIT = Path("/etc/systemd/system/electric-cherry-uninstall.service")
UNINSTALL_REQUEST = Path("/etc/electric-cherry/decommission.json")
ROOT_MARKER = ".electric-cherry.json"
class AgentError(Exception):
@@ -47,6 +54,97 @@ class Rejected(AgentError):
"""A definitive precondition failure, before any destructive action."""
def systemctl(*args):
try:
result = subprocess.run(["systemctl", *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60, check=False)
except (OSError, subprocess.TimeoutExpired):
raise AgentError("Host service management is unavailable") from None
if result.returncode:
raise AgentError("Host service management did not complete")
def protected_path(path, missing=False):
"""Deletion authority must never come from a runtime-writable path."""
path = Path(path)
for current in (*reversed(path.parents), path):
try:
info = current.lstat()
except FileNotFoundError:
if missing:
continue
raise AgentError("Host uninstall files are unavailable; reinstall the agent") from None
if stat_module.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
raise AgentError("Host uninstall paths must be owned and writable only by root")
return path
def protected_file(path, missing=False):
path = protected_path(path, missing=missing)
try:
if not stat_module.S_ISREG(path.lstat().st_mode):
raise AgentError("Host uninstall metadata must be a regular file")
except FileNotFoundError:
if not missing:
raise
return path
def sync_parent(path):
descriptor = os.open(Path(path).parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
def uninstall_root(value, identity=None):
"""Accept only a canonical, dedicated directory with no nested mounts."""
if not isinstance(value, str) or not re.fullmatch(r"/[A-Za-z0-9/_.-]+", value):
raise AgentError("Host data directory is unsafe to remove")
root = Path(value)
forbidden = {"/etc", "/usr", "/var", "/var/lib", "/var/log", "/var/cache", "/home", "/root", "/opt", "/srv", "/tmp", "/run", "/mnt", "/media", "/usr/local", "/usr/local/libexec"}
if str(root) != value or ".." in root.parts or "." in root.parts or len(root.parts) < 3 or value in forbidden:
raise AgentError("Host data directory is unsafe to remove")
for installed in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT, UNINSTALL_EXECUTABLE, UNINSTALL_UNIT):
if installed == root or root in installed.parents:
raise AgentError("Host data directory overlaps the agent installation")
protected_path(root, missing=True)
if root.exists():
info = root.stat()
if not stat_module.S_ISDIR(info.st_mode):
raise AgentError("Host data directory is not a directory")
if identity and [info.st_dev, info.st_ino] != identity:
raise AgentError("Host data directory changed since deletion was authorized")
try:
# ProtectSystem/ReadWritePaths give the running agent a private bind
# mount at its data directory. Inspect the host systemd namespace,
# otherwise every legitimate installation appears to be a mount root.
mount_lines = Path("/proc/1/mountinfo").read_text().splitlines()
except FileNotFoundError:
mount_lines = [] # Non-Linux tests; the installed service requires Linux.
for line in mount_lines:
fields = line.split()
if len(fields) < 5:
continue
mount = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), fields[4])
if mount == value or mount.startswith(value + "/"):
raise AgentError("Unmount filesystems inside the host data directory before deletion")
return root
def uninstall_ownership(config):
root = uninstall_root(config.get("root"))
marker = protected_file(root / ROOT_MARKER)
try:
saved = json.loads(marker.read_text())
except (OSError, ValueError):
raise AgentError("Host data ownership is unavailable; reinstall the agent") from None
expected = {"version": 1, "root": str(root), "hostId": config.get("hostId"), "controller": config.get("controller", "").rstrip("/")}
if saved != expected or not expected["hostId"]:
raise AgentError("Host data ownership does not match this enrollment")
return root
def digest(data):
return hashlib.sha256(data).hexdigest()
@@ -275,7 +373,7 @@ class Docker:
raise AgentError("Docker command unavailable or timed out") from None
if result.returncode:
# Never include arbitrary Docker stderr: create errors may echo secrets.
if tolerate_missing and (b"No such object" in (result.stderr or b"") or b"No such container" in (result.stderr or b"")):
if tolerate_missing and any(message in (result.stderr or b"") for message in (b"No such object", b"No such container", b"No such image")):
return None
raise AgentError("Docker rejected the requested operation")
return result.stdout.decode("utf-8", "replace")
@@ -285,6 +383,188 @@ class Docker:
return json.loads(raw)[0] if raw is not None else None
class Uninstaller:
"""A separate systemd service owns cleanup after stopping the host agent.
Its root-only request and executable stay on disk until the controller
acknowledges data removal. Completion is reported only after removing those
final files. A lost final response can be retried from memory; a power loss
in that last window deliberately leaves controller status unconfirmed.
"""
def __init__(self, request_path, docker=None):
if Path(request_path) != UNINSTALL_REQUEST:
raise AgentError("Invalid host uninstall request location")
protected_file(UNINSTALL_REQUEST)
self.job = json.loads(UNINSTALL_REQUEST.read_text())
self.docker = docker or Docker()
self.root = uninstall_root(self.job.get("root"), self.job.get("rootIdentity"))
host_id = identifier(self.job.get("hostId"))
operation_id = identifier(self.job.get("operationId"))
parsed = urllib.parse.urlsplit(self.job.get("controller", ""))
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment:
raise AgentError("Host uninstall controller must use HTTPS")
if self.job.get("receiptUrl") != "/api/agent/decommission/" + operation_id or not isinstance(self.job.get("receiptToken"), str) or not self.job["receiptToken"]:
raise AgentError("Invalid host uninstall receipt")
if not isinstance(self.job.get("rootIdentity"), list) or len(self.job["rootIdentity"]) != 2 or not all(isinstance(value, int) and not isinstance(value, bool) for value in self.job["rootIdentity"]):
raise AgentError("Invalid host data ownership proof")
if self.root.exists():
# The authorized device/inode is persisted before deletion so a
# crash halfway through rmtree does not require a deleted marker.
if not self.job.get("removalStarted"):
uninstall_ownership({"root": str(self.root), "controller": self.job["controller"], "hostId": host_id})
def save(self, **changes):
self.job.update(changes)
atomic_json(UNINSTALL_REQUEST, self.job)
def receipt(self, stage, error=None):
payload = {"stage": stage}
if error:
payload["error"] = error
request = urllib.request.Request(self.job["controller"].rstrip("/") + self.job["receiptUrl"], data=json.dumps(payload).encode(), headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.job["receiptToken"], "User-Agent": "electric-cherry-agent/" + VERSION}, method="POST")
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, hdrs, newurl):
raise AgentError("Controller redirects are not allowed")
try:
with urllib.request.build_opener(NoRedirect()).open(request, timeout=35) as response:
if json.load(response).get("acknowledged") is not True:
raise AgentError("Controller has not acknowledged host removal")
except (OSError, ValueError):
raise AgentError("Host removal receipt is waiting for controller contact") from None
def containers(self):
ids = self.docker.call("container", "ls", "--all", "--quiet", "--no-trunc").splitlines()
values = []
for container_id in ids:
info = self.docker.inspect(container_id)
if info:
values.append(info)
return values
def owned_container(self, info):
labels = info.get("Config", {}).get("Labels", {}) or {}
if labels.get(LABEL + ".managed") != "true":
return False
if labels.get(LABEL + ".host") not in (None, self.job["hostId"]):
return False
try:
instance_id, generation = instance_identity(labels.get(LABEL + ".instance"), int(labels.get(LABEL + ".generation", "")))
except (AgentError, ValueError, TypeError):
return False
expected = str(self.root / "instances" / instance_id / str(generation) / "app")
# Legacy agents did not label the host; its exact managed bind mount
# proves ownership without touching another installation's containers.
return any(mount.get("Type") == "bind" and mount.get("Source") == expected and mount.get("Destination") == "/home/plutainer/app" for mount in info.get("Mounts", []))
def uses_root(self, info):
for mount in info.get("Mounts", []):
if mount.get("Type") != "bind":
continue
source = Path(mount.get("Source", "/")).resolve()
if source == self.root or self.root in source.parents:
return True
return False
def remove_docker_resources(self):
containers = self.containers()
if any(self.uses_root(info) and not self.owned_container(info) for info in containers):
raise AgentError("An unrelated container uses the host data directory; detach it before retrying deletion")
owned = [info for info in containers if self.owned_container(info)]
if any(info.get("State", {}).get("Running") is not False or info.get("State", {}).get("Status") not in ("created", "exited", "dead") for info in owned):
raise AgentError("A managed game container is still running or its state is unknown; retire it before deleting this host")
for info in owned:
# Do not force removal: Docker must also reject a container that
# starts again after inspection rather than interrupting players.
self.docker.call("container", "rm", "--volumes", info["Id"], timeout=120, tolerate_missing=True)
remaining = self.containers()
if any(self.uses_root(info) for info in remaining):
raise AgentError("Containers still use the host data directory")
# Never prune Docker globally. Only remove this agent's recorded image
# references, and preserve images used by any unrelated container.
used_images = {info.get("Image") for info in remaining}
images = self.job.get("runtimeImages", {})
if not isinstance(images, dict):
raise AgentError("Managed runtime image ownership is unavailable")
for reference, expected_id in images.items():
if not isinstance(reference, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{0,511}", reference) or not isinstance(expected_id, str):
raise AgentError("Managed runtime image ownership is invalid")
raw = self.docker.call("image", "inspect", reference, tolerate_missing=True)
if raw is None:
continue
found = json.loads(raw)
if not found or found[0].get("Id") != expected_id or expected_id in used_images:
continue
self.docker.call("image", "rm", reference, tolerate_missing=True)
def remove_data(self):
protected_path(AGENT_CONFIG.parent, missing=True)
if not self.job.get("agentStopped"):
systemctl("disable", "--now", AGENT_UNIT.name)
self.save(agentStopped=True)
self.remove_docker_resources()
root = uninstall_root(str(self.root), self.job["rootIdentity"])
# This checkpoint also authorizes safe continuation if a power loss
# removes the ownership marker before the rest of the tree.
self.save(removalStarted=True)
if root.exists():
if not shutil.rmtree.avoids_symlink_attacks:
raise AgentError("This platform cannot safely remove managed host files")
shutil.rmtree(root)
sync_parent(root)
for path in (AGENT_CONFIG, AGENT_EXECUTABLE, AGENT_UNIT):
protected_file(path, missing=True)
path.unlink(missing_ok=True)
sync_parent(path)
systemctl("daemon-reload")
self.save(dataRemoved=True)
def remove_helper(self):
# Disable does not stop this process: it must outlive its own unit and
# executable to deliver the final receipt after the files are gone.
protected_file(UNINSTALL_UNIT, missing=True)
if UNINSTALL_UNIT.exists():
systemctl("disable", UNINSTALL_UNIT.name)
UNINSTALL_UNIT.unlink()
sync_parent(UNINSTALL_UNIT)
systemctl("daemon-reload")
protected_file(UNINSTALL_EXECUTABLE, missing=True)
UNINSTALL_EXECUTABLE.unlink(missing_ok=True)
sync_parent(UNINSTALL_EXECUTABLE)
protected_path(UNINSTALL_REQUEST.parent, missing=True)
if UNINSTALL_REQUEST.parent.exists():
shutil.rmtree(UNINSTALL_REQUEST.parent)
sync_parent(UNINSTALL_REQUEST.parent)
def run(self):
if not self.job.get("dataRemoved"):
try:
self.remove_data()
except (AgentError, OSError, ValueError, KeyError) as error:
with contextlib.suppress(AgentError):
self.receipt("failed", str(error) if isinstance(error, AgentError) else "Host cleanup could not finish. Check the uninstall service and resolve filesystem errors; cleanup will retry.")
raise
self.receipt("cleaned")
while True:
try:
self.remove_helper()
break
except (AgentError, OSError, ValueError):
# Once its unit/executable has been unlinked systemd can no
# longer restart this worker. Finish idempotently in memory.
with contextlib.suppress(AgentError):
self.receipt("failed", "Host files were removed, but removing the uninstall service is still retrying")
time.sleep(15)
# The controller never marks a host deleted on the earlier receipt.
# No credentials or retry state are left on disk at this point.
while True:
try:
self.receipt("completed")
return
except AgentError:
time.sleep(15)
class Agent:
def __init__(self, config, docker=None):
self.config = config
@@ -300,6 +580,43 @@ class Agent:
self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
self.executing_id = None
def can_uninstall(self):
try:
if Path(self.config_path) != AGENT_CONFIG:
return False
uninstall_ownership(self.config)
protected_file(UNINSTALL_EXECUTABLE)
protected_file(UNINSTALL_UNIT)
return True
except (AgentError, OSError, ValueError):
return False
def decommission(self, operation):
payload = operation.get("payload", {})
if payload.get("hostId") != self.config.get("hostId") or payload.get("operationId") != operation["id"]:
raise Rejected("Host removal identity does not match this agent")
if payload.get("receiptUrl") != "/api/agent/decommission/" + identifier(operation["id"]) or not isinstance(payload.get("receiptToken"), str) or not payload["receiptToken"]:
raise Rejected("Invalid host removal receipt")
if not self.can_uninstall():
raise Rejected("Reinstall the agent to enable safe removal of its dedicated host files")
root = uninstall_ownership(self.config)
info = root.stat()
protected_file(UNINSTALL_REQUEST, missing=True)
aliases_path = root / "agent" / "runtime-images.json"
protected_file(aliases_path, missing=True)
aliases = json.loads(aliases_path.read_text()) if aliases_path.exists() else {}
job = {"hostId": payload["hostId"], "operationId": operation["id"], "receiptUrl": payload["receiptUrl"], "receiptToken": payload["receiptToken"], "controller": self.config["controller"].rstrip("/"), "root": str(root), "rootIdentity": [info.st_dev, info.st_ino], "runtimeImages": aliases}
if UNINSTALL_REQUEST.exists():
previous = json.loads(UNINSTALL_REQUEST.read_text())
if any(previous.get(key) != job[key] for key in ("hostId", "operationId", "receiptUrl", "receiptToken", "controller", "root", "rootIdentity")):
raise Rejected("A different host removal is already pending")
self.checkpoint({"phase": "uninstall-service"})
if not UNINSTALL_REQUEST.exists():
atomic_json(UNINSTALL_REQUEST, job)
systemctl("enable", "--now", "--no-block", UNINSTALL_UNIT.name)
self.stopping.set()
return {"decommissioning": True, "message": "The uninstall service will report completion after host files and the agent are removed"}
def instance_dir(self, instance_id, generation):
instance_id, generation = instance_identity(instance_id, generation)
return self.root / "instances" / instance_id / str(generation)
@@ -416,7 +733,7 @@ class Agent:
for line in Path("/etc/os-release").read_text().splitlines():
if line.startswith("PRETTY_NAME="):
os_name = line.split("=", 1)[1].strip('"')
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": ["host.decommission"] if self.can_uninstall() else [], "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
def safe_to_disrupt(self, instance_id, generation):
health = self.inspect(instance_id, generation)
@@ -525,6 +842,8 @@ class Agent:
name = self.container_name(instance_id, generation)
spec_sha = digest(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode())
args = ["container", "create", "--name", name, "--restart", "unless-stopped", "--label", LABEL + ".managed=true", "--label", LABEL + ".instance=" + instance_id, "--label", LABEL + ".generation=" + str(generation), "--label", LABEL + ".operation=" + (self.executing_id or "direct"), "--label", LABEL + ".spec-sha256=" + spec_sha, "--mount", f"type=bind,src={app},dst=/home/plutainer/app", "--mount", f"type=bind,src={library / 'content'},dst=/home/plutainer/gamefiles,readonly", "--env-file", str(env_path)]
if self.config.get("hostId"):
args.extend(["--label", LABEL + ".host=" + identifier(self.config["hostId"])])
bind_arg = "[" + bind + "]" if ":" in bind else bind
for protocol in ("udp", "tcp"):
args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
@@ -923,6 +1242,8 @@ class Agent:
kind, payload = operation.get("kind"), operation.get("payload", {})
if not isinstance(kind, str) or not isinstance(payload, dict):
raise AgentError("Invalid operation")
if kind == "host.decommission":
return self.decommission(operation)
if kind == "credentials.rotate":
pending = "ec_host_" + base64.urlsafe_b64encode(os.urandom(32)).rstrip(b"=").decode()
self.config["pendingCredential"] = pending
@@ -997,7 +1318,7 @@ class Agent:
return self.journal.update(operation["id"], "failed" if isinstance(err, Rejected) or not checkpoint else "unknown", error=message)
finally:
self.executing_id = None
return self.journal.update(operation["id"], "completed", result)
return self.journal.update(operation["id"], "running" if operation["kind"] == "host.decommission" else "completed", result)
def recover_credential(self):
pending = self.config.get("pendingCredential")
@@ -1024,6 +1345,13 @@ class Agent:
instance_id, generation = operation.get("instanceId"), operation.get("generation")
checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
try:
if kind == "host.decommission":
self.executing_id = operation["id"]
try:
result = self.decommission(operation)
return self.journal.update(operation["id"], "running", result)
finally:
self.executing_id = None
if kind == "credentials.rotate":
if not self.config.get("pendingCredential") or self.recover_credential():
return self.journal.update(operation["id"], "completed", {"rotated": True, "reconciled": True})
@@ -1183,8 +1511,14 @@ def main():
parser.add_argument("--token-file")
parser.add_argument("--root", default="/var/lib/electric-cherry")
parser.add_argument("--force-enroll", action="store_true")
parser.add_argument("--uninstall", metavar="REQUEST", help="Run the installed host removal service")
parser.add_argument("--version", action="version", version=VERSION)
args = parser.parse_args()
if args.uninstall:
if os.geteuid() != 0 or args.enroll:
raise AgentError("Host uninstall must run as the dedicated root service")
Uninstaller(args.uninstall).run()
return
if args.enroll:
if not args.controller or not args.token_file:
parser.error("Enrollment needs --controller and --token-file")
+120 -7
View File
@@ -21,7 +21,7 @@ done
case "$controller" in https://*) ;; *) echo '--controller must be an HTTPS URL.' >&2; exit 2 ;; esac
[ -n "$enrollment_token" ] || { echo '--token is required.' >&2; exit 2; }
case "$data_dir" in /*) ;; *) echo '--data-dir must be absolute.' >&2; exit 2 ;; esac
case "$data_dir" in *[!a-zA-Z0-9/_.-]*|/|/etc|/usr|/var|/var/lib) echo 'Choose a dedicated data directory.' >&2; exit 2 ;; esac
case "$data_dir" in *[!a-zA-Z0-9/_.-]*|*/../*|*/./*|*/..|*/.|*//*|*/) echo 'Choose a canonical, dedicated data directory.' >&2; exit 2 ;; esac
[ ! -L "$data_dir" ] || { echo 'Data directory must not be a symlink.' >&2; exit 2; }
. /etc/os-release
[ "${ID:-}" = debian ] || { echo 'Supported platform: Debian 12 or 13.' >&2; exit 1; }
@@ -32,13 +32,82 @@ command -v systemctl >/dev/null || { echo 'systemd is required.' >&2; exit 1; }
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y --no-install-recommends ca-certificates curl python3
python3 - "$controller" "$data_dir" <<'PY'
import json, pathlib, sys
config = pathlib.Path('/etc/electric-cherry/agent.json')
if config.exists():
saved = json.loads(config.read_text())
if saved.get('controller', '').rstrip('/') != sys.argv[1].rstrip('/') or pathlib.Path(saved.get('root', '')).resolve() != pathlib.Path(sys.argv[2]).resolve():
python3 - "$controller" "$data_dir" /etc/electric-cherry/agent.json <<'PY'
# INSTALLATION_PREFLIGHT: this runs before touching managed directories or services.
import json, pathlib, re, stat, sys
controller, raw_root, config_name = sys.argv[1:]
root = pathlib.Path(raw_root)
config = pathlib.Path(config_name)
reserved = {
'/', '/bin', '/boot', '/dev', '/etc', '/home', '/lib', '/lib64',
'/media', '/mnt', '/opt', '/proc', '/root', '/run', '/sbin', '/srv',
'/sys', '/tmp', '/usr', '/usr/local', '/usr/local/bin', '/usr/local/lib',
'/usr/local/libexec', '/var', '/var/cache', '/var/lib', '/var/log', '/var/tmp',
}
if (not re.fullmatch(r'/[A-Za-z0-9/_.-]+', raw_root)
or str(root) != raw_root or any(part in ('.', '..') for part in raw_root.split('/')[1:])
or len(root.parts) < 3 or raw_root in reserved or config.parent == root or config.parent in root.parents):
raise SystemExit('Choose a canonical, dedicated data directory outside the agent configuration directory.')
installed_paths = [config,
pathlib.Path('/usr/local/libexec/electric-cherry-agent.py'),
pathlib.Path('/usr/local/libexec/electric-cherry-uninstall.py'),
pathlib.Path('/etc/systemd/system/electric-cherry-agent.service'),
pathlib.Path('/etc/systemd/system/electric-cherry-uninstall.service')]
if any(root == path or root in path.parents for path in installed_paths):
raise SystemExit('Choose a dedicated data directory outside the agent executable and service directories.')
def trusted_directories(path):
for current in [*reversed(path.parents), path]:
try:
info = current.lstat()
except FileNotFoundError:
continue
if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
raise SystemExit('Installation directories and their parents must be root-owned, not symlinks, and not writable by other users.')
def trusted_json(path):
info = path.lstat()
if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
raise SystemExit('Existing installation metadata must be regular, root-owned files that other users cannot modify.')
value = json.loads(path.read_text())
if not isinstance(value, dict):
raise SystemExit('Existing installation metadata is invalid.')
return value
trusted_directories(root)
trusted_directories(config.parent)
mountinfo = pathlib.Path('/proc/1/mountinfo')
if mountinfo.exists():
for line in mountinfo.read_text().splitlines():
fields = line.split()
if len(fields) < 5:
continue
mount = re.sub(r'\\([0-7]{3})', lambda match: chr(int(match[1], 8)), fields[4])
if mount == raw_root or mount.startswith(raw_root + '/'):
raise SystemExit('Use a dedicated subdirectory of the mounted disk, with no filesystems mounted inside it, so host deletion can remove the entire data directory.')
request = config.with_name('decommission.json')
if request.exists() or request.is_symlink():
raise SystemExit('Host deletion is in progress. Finish or repair that deletion before installing again.')
saved = None
if config.exists() or config.is_symlink():
saved = trusted_json(config)
if saved.get('controller', '').rstrip('/') != controller.rstrip('/') or saved.get('root') != raw_root:
raise SystemExit('Existing enrollment uses a different controller or data directory. Preserve the existing installation and migrate it explicitly.')
if not saved.get('hostId') or not saved.get('credential'):
raise SystemExit('Existing enrollment is incomplete. Repair its credentials before upgrading.')
marker = root / '.electric-cherry.json'
if marker.exists() or marker.is_symlink():
owned = trusted_json(marker)
if not saved or owned != {'version': 1, 'root': raw_root, 'hostId': saved['hostId'], 'controller': controller.rstrip('/')}:
raise SystemExit('Data directory ownership does not match this enrollment. Preserve it and migrate explicitly.')
elif root.exists() and any(root.iterdir()):
# Upgrade installations predating ownership markers only when the entire
# root has the known layout. Never adopt a directory of unrelated files.
if not saved or any(path.name not in {'agent', 'instances', 'libraries'} for path in root.iterdir()):
raise SystemExit('The data directory contains files not owned by this installation. Choose an empty, dedicated directory.')
for directory in ('agent', 'instances', 'libraries'):
trusted_directories(root / directory)
PY
if ! command -v docker >/dev/null; then
if [ -e /var/lib/docker ] || [ -e /etc/docker/daemon.json ]; then
@@ -76,9 +145,36 @@ install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-
install -d -m 0755 /usr/local/libexec
install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-agent.py
}
# A separate worker survives stopping/removing the agent service. Its only
# entry point is a fixed, authenticated deletion request prepared by the agent.
install -m 0755 "$temporary_dir/ec-agent.py" /usr/local/libexec/electric-cherry-uninstall.py
printf '%s' "$enrollment_token" > "$temporary_dir/enrollment-token"
unset enrollment_token
python3 /usr/local/libexec/electric-cherry-agent.py --enroll --controller "$controller" --root "$data_dir" --token-file "$temporary_dir/enrollment-token" ${reenroll:+"$reenroll"}
python3 - "$data_dir" /etc/electric-cherry/agent.json <<'PY'
# INSTALLATION_OWNERSHIP: bind deletion authority to this exact enrollment/root.
import json, os, pathlib, sys, uuid
root = pathlib.Path(sys.argv[1])
config = json.loads(pathlib.Path(sys.argv[2]).read_text())
if config.get('root') != str(root) or not config.get('hostId') or not config.get('controller'):
raise SystemExit('Enrollment did not establish ownership of this data directory.')
marker = root / '.electric-cherry.json'
temporary = root / ('.electric-cherry.' + uuid.uuid4().hex + '.tmp')
try:
with temporary.open('x') as stream:
os.chmod(temporary, 0o600)
json.dump({'version': 1, 'root': str(root), 'hostId': config['hostId'], 'controller': config['controller'].rstrip('/')}, stream)
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, marker)
directory_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
finally:
temporary.unlink(missing_ok=True)
PY
cat > /etc/systemd/system/electric-cherry-agent.service <<EOF
[Unit]
Description=Electric Cherry Host Agent
@@ -99,6 +195,23 @@ ProtectHome=true
PrivateTmp=true
ReadWritePaths=$data_dir /etc/electric-cherry
[Install]
WantedBy=multi-user.target
EOF
cat > /etc/systemd/system/electric-cherry-uninstall.service <<'EOF'
[Unit]
Description=Remove an Electric Cherry Host Installation
After=network-online.target docker.service
Wants=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/bin/python3 /usr/local/libexec/electric-cherry-uninstall.py --uninstall /etc/electric-cherry/decommission.json
Restart=on-failure
RestartSec=15
TimeoutStartSec=0
UMask=0077
[Install]
WantedBy=multi-user.target
EOF
+2 -2
View File
@@ -1,6 +1,6 @@
{
"version": "1.0.0",
"version": "1.1.0",
"protocolVersion": 1,
"platform": "debian-amd64",
"sha256": "2b6e6c2cc4c949e8f980c147aa6b8725844b7c320d7627e0328570d23c985a43"
"sha256": "67046a3d81f251e148f2323cade255ff584a622750a8c05ee27f96c70e09fcc7"
}
+4 -2
View File
@@ -46,10 +46,11 @@ describe('management route permissions', () => {
});
it.each([
['/api/hosts', 'POST'], ['/api/hosts/[id]', 'PATCH'], ['/api/hosts/[id]/enrollment', 'POST'],
['/api/hosts', 'POST'], ['/api/hosts/[id]', 'PATCH'], ['/api/hosts/[id]', 'DELETE'], ['/api/hosts/[id]/enrollment', 'POST'],
['/api/hosts/[id]/credentials', 'DELETE'], ['/api/hosts/[id]/credentials', 'POST'],
['/api/hosts/[id]/operations', 'POST'], ['/api/settings', 'PUT'], ['/api/game-files', 'POST'],
['/api/game-files/[id]/download', 'POST'], ['/api/maintenance', 'POST'], ['/api/watchdog', 'POST'], ['/api/plutonium', 'POST']
['/api/game-files/[id]/download', 'POST'], ['/api/game-files/[id]/upload/files/[...path]', 'PUT'],
['/api/game-files/[id]/upload/complete', 'POST'], ['/api/maintenance', 'POST'], ['/api/watchdog', 'POST'], ['/api/plutonium', 'POST']
])('protects host, game-file, and global policy mutations at %s', (route, method) => {
expect(routePermission(route, method)).toBe('admin');
expect(allows('operator', route, method)).toBe(false);
@@ -91,6 +92,7 @@ describe('authentication and machine boundaries', () => {
it.each([
['/api/agent/enroll', 'POST'], ['/api/agent/heartbeat', 'POST'], ['/api/agent/operations', 'GET'],
['/api/agent/operations/[id]', 'POST'], ['/api/agent/libraries/[id]/files/[...path]', 'GET'],
['/api/agent/decommission/[id]', 'POST'],
['/api/agent/imports/[id]/files/[...path]', 'PUT'], ['/api/agent/imports/[id]/complete', 'POST'],
['/api/agent/install.sh', 'GET'], ['/api/agent/release/[file]', 'GET'], ['/api/telemetry', 'POST']
])('preserves endpoint-specific machine authentication for %s', (route, method) => {
+227
View File
@@ -0,0 +1,227 @@
import Database from 'better-sqlite3';
import { createHash } from 'node:crypto';
import { mkdir, mkdtemp, readFile, rename, rm, stat } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const state = vi.hoisted(() => ({ handle: null as Database.Database | null }));
vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle }));
import {
browserUploadStatus, createLibraryImport, createLibraryUpload, getHostLibraryCache,
getLibraryVersion, listLibraries, publishBrowserUpload, queueLibraryDownload,
requestBrowserPublication, requireReadyLibrary, updateHostLibraryCaches, uploadBrowserFile
} from '../../src/lib/server/game-files/index.js';
import { createEnrollment, createHost, enrollHost, listHosts } from '../../src/lib/server/hosts/index.js';
import { manifestChecksum } from '../../src/lib/server/game-files/storage.js';
import { serveGameFile } from '../../src/lib/server/game-files/http.js';
import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js';
let directory: string;
const modified = 1_700_000_000_000;
const bytes = Buffer.from('verified Plutonium game content');
const sha = (value: Uint8Array) => createHash('sha256').update(value).digest('hex');
const file = (path = 'zone/base.ff', content = bytes) => ({ path, size: content.length, lastModified: modified });
const upload = (id: string, path: string, content: Buffer, offset = 0, checksum = sha(content)) =>
uploadBrowserFile(id, path, { offset, size: content.length, sha256: checksum }, new Response(new Uint8Array(content)).body);
const begin = (game: 't5' | 't6' = 't6', files = [file()], version = '1') =>
createLibraryUpload({ game, name: `Plutonium ${game.toUpperCase()}`, version, description: 'Uploaded game folder', files });
beforeEach(async () => {
directory = await mkdtemp(join(tmpdir(), 'ec-browser-library-'));
state.handle = new Database(join(directory, 'controller.sqlite'));
state.handle.pragma('foreign_keys=ON');
vi.stubEnv('GAME_FILES_ROOT', join(directory, 'libraries'));
});
afterEach(async () => {
state.handle?.close(); state.handle = null;
await rm(directory, { recursive: true, force: true });
vi.unstubAllEnvs();
});
function host() {
const created = createHost({ name: 'Fresh download target' });
const { credential } = enrollHost(createEnrollment(created.id).token, {
protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3
});
return { id: created.id, credential };
}
describe('browser folder uploads', () => {
it.each(['t5', 't6'] as const)('publishes %s from a browser without an enrolled host or DLC declarations', async game => {
expect(listHosts()).toEqual([]);
const initial = begin(game);
expect(initial).toMatchObject({ state: 'importing', operationId: null, sourceHostId: null, sourceVolume: null });
expect(initial).not.toHaveProperty('dlc');
expect(await browserUploadStatus(initial.id)).toMatchObject({
status: 'pending', files: [{ ...file(), receivedBytes: 0, complete: false }]
});
await upload(initial.id, file().path, bytes);
const accepted = await requestBrowserPublication(initial.id);
expect(['verifying', 'published']).toContain(accepted.status);
const ready = await publishBrowserUpload(initial.id);
expect(ready).toMatchObject({ state: 'ready', sizeBytes: bytes.length, fileCount: 1 });
expect(ready.files).toEqual([{ path: file().path, size: bytes.length, sha256: sha(bytes) }]);
expect(ready.manifestSha256).toBe(manifestChecksum(ready.files));
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
expect(await browserUploadStatus(initial.id)).toMatchObject({ status: 'published' });
expect(listLibraries()[0].versions[0]).not.toHaveProperty('dlc');
expect(listHosts()).toEqual([]);
});
it('preserves nested relative contents and derives a canonical manifest in path order', async () => {
const selected = [file('zone/english/a.ff'), file('main/iw_00.iwd'), file('empty.txt', Buffer.alloc(0))];
const initial = begin('t6', selected);
for (const item of selected) await upload(initial.id, item.path, item.size ? bytes : Buffer.alloc(0));
const ready = await publishBrowserUpload(initial.id);
expect(ready.files.map(item => item.path)).toEqual(['empty.txt', 'main/iw_00.iwd', 'zone/english/a.ff']);
expect(ready.files[0].sha256).toBe(sha(Buffer.alloc(0)));
expect(ready.manifestSha256).toBe(manifestChecksum(ready.files));
expect(await readFile(join(ready.storageLocation, 'zone/english/a.ff'))).toEqual(bytes);
});
it('rejects incomplete publication and allows the remaining files to finish the same version', async () => {
const initial = begin('t5', [file('zone/first.ff'), file('zone/second.ff')]);
await upload(initial.id, 'zone/first.ff', bytes);
await expect(publishBrowserUpload(initial.id)).rejects.toThrow();
expect(getLibraryVersion(initial.id)?.state).not.toBe('ready');
const hidden = await serveGameFile(new Request('https://ec.test/files'), initial.id, 'zone/first.ff');
expect(hidden.status).toBe(404);
await upload(initial.id, 'zone/second.ff', bytes);
const ready = await publishBrowserUpload(initial.id);
expect(ready.state).toBe('ready');
expect(ready.id).toBe(initial.id);
});
it('does not acknowledge corrupt chunks and permits a verified retry', async () => {
const initial = begin();
await expect(upload(initial.id, file().path, bytes, 0, '0'.repeat(64))).rejects.toThrow();
expect((await browserUploadStatus(initial.id)).files[0]).toMatchObject({ receivedBytes: 0, complete: false });
await expect(uploadBrowserFile(initial.id, file().path, {
offset: 0, size: bytes.length, sha256: sha(bytes)
}, new Response(bytes.subarray(0, -1)).body)).rejects.toThrow();
expect((await browserUploadStatus(initial.id)).files[0].receivedBytes).toBe(0);
await upload(initial.id, file().path, bytes);
expect((await browserUploadStatus(initial.id)).files[0]).toMatchObject({ receivedBytes: bytes.length, complete: true });
expect((await publishBrowserUpload(initial.id)).state).toBe('ready');
});
it('surfaces asynchronous incomplete publication as retryable failure', async () => {
const initial = begin();
await requestBrowserPublication(initial.id);
await vi.waitFor(async () => expect((await browserUploadStatus(initial.id)).status).toBe('failed'));
expect((await browserUploadStatus(initial.id)).error).toBeTruthy();
await upload(initial.id, file().path, bytes);
expect(await browserUploadStatus(initial.id)).toMatchObject({ status: 'pending', error: null });
await requestBrowserPublication(initial.id);
expect((await publishBrowserUpload(initial.id)).state).toBe('ready');
});
it('resumes from the verified chunk boundary and safely retries a lost acknowledgement', async () => {
const sizeProbe = begin();
const chunkSize = (await browserUploadStatus(sizeProbe.id)).chunkSize;
expect(chunkSize).toBeGreaterThan(0);
const large = Buffer.alloc(chunkSize + 17, 0x41);
const initial = begin('t6', [file('main/archive.iwd', large)], 'large');
const first = large.subarray(0, chunkSize);
await upload(initial.id, 'main/archive.iwd', first);
expect((await browserUploadStatus(initial.id)).files[0]).toMatchObject({ receivedBytes: chunkSize, complete: false });
state.handle!.close();
state.handle = new Database(join(directory, 'controller.sqlite'));
state.handle.pragma('foreign_keys=ON');
await expect(upload(initial.id, 'main/archive.iwd', first)).rejects.toMatchObject({ receivedBytes: chunkSize });
expect((await browserUploadStatus(initial.id)).files[0].receivedBytes).toBe(chunkSize);
await upload(initial.id, 'main/archive.iwd', large.subarray(chunkSize), chunkSize);
const ready = await publishBrowserUpload(initial.id);
expect(ready.files[0].sha256).toBe(sha(large));
expect((await readFile(join(ready.storageLocation, 'main/archive.iwd'))).equals(large)).toBe(true);
});
it('rejects unknown file paths, traversal, and chunks outside the declared file', async () => {
const initial = begin();
await expect(upload(initial.id, 'zone/unselected.ff', bytes)).rejects.toThrow();
await expect(upload(initial.id, '../escape.ff', bytes)).rejects.toThrow();
await expect(upload(initial.id, file().path, bytes, bytes.length + 1)).rejects.toThrow();
await expect(upload(initial.id, file().path, Buffer.concat([bytes, bytes]))).rejects.toThrow();
expect((await browserUploadStatus(initial.id)).files[0].receivedBytes).toBe(0);
});
it('keeps ready bytes and version metadata immutable and requires a new version label', async () => {
const initial = begin(); await upload(initial.id, file().path, bytes);
const ready = await publishBrowserUpload(initial.id);
await expect(upload(initial.id, file().path, bytes)).rejects.toThrow();
expect(() => state.handle!.prepare('UPDATE game_file_versions SET description=? WHERE id=?').run('changed', ready.id)).toThrow(/immutable/);
expect(() => begin()).toThrow(/already exists/);
expect((await publishBrowserUpload(initial.id)).manifestSha256).toBe(ready.manifestSha256);
expect(begin('t6', [file()], '2').id).not.toBe(ready.id);
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
});
it('recovers publication after the verified directory was renamed but the ready commit was interrupted', async () => {
const initial = begin(); await upload(initial.id, file().path, bytes);
const files = [{ path: file().path, size: bytes.length, sha256: sha(bytes) }];
const checksum = manifestChecksum(files);
state.handle!.transaction(() => {
state.handle!.prepare('UPDATE game_file_versions SET manifest=?,manifest_sha256=? WHERE id=?')
.run(JSON.stringify(files), checksum, initial.id);
state.handle!.prepare('INSERT INTO game_file_entries(version_id,path,size,sha256) VALUES(?,?,?,?)')
.run(initial.id, file().path, bytes.length, sha(bytes));
})();
await mkdir(dirname(initial.storageLocation), { recursive: true });
await rename(join(directory, 'libraries', '.imports', initial.id), initial.storageLocation);
state.handle!.close(); state.handle = new Database(join(directory, 'controller.sqlite'));
state.handle.pragma('foreign_keys=ON');
expect(await browserUploadStatus(initial.id)).toMatchObject({ status: 'pending', files: [{ complete: true }] });
const ready = await publishBrowserUpload(initial.id);
expect(ready).toMatchObject({ id: initial.id, state: 'ready', manifestSha256: checksum, files });
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
});
it('rejects chunks racing with final publication without recreating staging files', async () => {
const initial = begin(); await upload(initial.id, file().path, bytes);
const publishing = publishBrowserUpload(initial.id);
await expect(upload(initial.id, file().path, bytes)).rejects.toMatchObject({ status: 409 });
const ready = await publishing;
expect(ready.state).toBe('ready');
expect(await readFile(join(ready.storageLocation, file().path))).toEqual(bytes);
await expect(stat(join(directory, 'libraries', '.imports', initial.id))).rejects.toMatchObject({ code: 'ENOENT' });
});
it('rejects browser operations against an existing host-sourced import', async () => {
const source = host();
const legacy = createLibraryImport({ hostId: source.id, game: 't6', name: 'Legacy import', version: '1' });
expect(legacy.sourceHostId).toBe(source.id);
await expect(Promise.resolve().then(() => browserUploadStatus(legacy.id))).rejects.toThrow();
await expect(upload(legacy.id, file().path, bytes)).rejects.toThrow();
await expect(publishBrowserUpload(legacy.id)).rejects.toThrow();
await expect(Promise.resolve().then(() => requestBrowserPublication(legacy.id))).rejects.toThrow();
expect(getLibraryVersion(legacy.id)?.state).toBe('importing');
});
it('distributes uploaded versions through the unchanged agent manifest, ranges, and verified cache contract', async () => {
const initial = begin('t5'); await upload(initial.id, file().path, bytes);
const ready = await publishBrowserUpload(initial.id);
const target = host();
const operation = queueLibraryDownload(target.id, ready.id);
expect(operation).toMatchObject({ kind: 'library.download', payload: {
libraryVersionId: ready.id, game: 't5', manifestSha256: ready.manifestSha256,
manifestUrl: `/api/agent/libraries/${ready.id}/manifest`, filesBaseUrl: `/api/agent/libraries/${ready.id}/files`
} });
const manifest = manifestRoute({ request: new Request(`https://ec.test/api/agent/libraries/${ready.id}/manifest`, {
headers: { Authorization: `Bearer ${target.credential}` }
}), params: { id: ready.id } });
expect(manifest.status).toBe(200);
expect(await manifest.json()).toEqual({ id: ready.id, game: 't5', files: ready.files, sizeBytes: bytes.length, manifestSha256: ready.manifestSha256 });
const range = await serveGameFile(new Request('https://ec.test/file', { headers: { Range: 'bytes=9-' } }), ready.id, file().path);
expect(range.status).toBe(206);
expect(Buffer.from(await range.arrayBuffer())).toEqual(bytes.subarray(9));
expect(() => requireReadyLibrary(target.id, ready.id)).toThrow(/finish downloading/);
updateHostLibraryCaches(target.id, [{ libraryVersionId: ready.id, status: 'ready', manifestSha256: '0'.repeat(64) }]);
expect(getHostLibraryCache(target.id, ready.id)?.status).toBe('failed');
updateHostLibraryCaches(target.id, [{ libraryVersionId: ready.id, status: 'ready', manifestSha256: ready.manifestSha256 }]);
expect(requireReadyLibrary(target.id, ready.id).id).toBe(ready.id);
const replacement = begin('t5', [file()], '2'); await upload(replacement.id, file().path, bytes);
await publishBrowserUpload(replacement.id);
expect(requireReadyLibrary(target.id, ready.id).manifestSha256).toBe(ready.manifestSha256);
});
});
+180
View File
@@ -0,0 +1,180 @@
import { createHash } from 'node:crypto';
import { mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import {
getUploadFileProgress, hashUploadedFiles, MAX_UPLOAD_CHUNK_BYTES,
receiveUploadChunk, UploadOffsetError, validateUploadPlan, withUploadLock
} from '../../src/lib/server/game-files/uploads.js';
import { manifestChecksum, verifyGameFile } from '../../src/lib/server/game-files/storage.js';
const sha = (value: string | Uint8Array) => createHash('sha256').update(value).digest('hex');
const stream = (value: string) => new Response(value).body!;
const file = { path: 'zone/english/map.ff', size: 10, lastModified: 1234 };
const chunk = (value: string, offset = 0) => ({ offset, size: Buffer.byteLength(value), sha256: sha(value) });
let directory: string;
let root: string;
beforeEach(async () => {
directory = await mkdtemp(join(tmpdir(), 'ec-folder-upload-'));
root = join(directory, 'import');
});
afterEach(async () => { await rm(directory, { recursive: true, force: true }); });
describe('browser folder upload plans', () => {
it('preserves folder structure and source identity while sorting canonical paths', () => {
expect(validateUploadPlan([{ path: 'zone/map.ff', size: 10, lastModified: 100 }, { path: 'main/data.iwd', size: 0 }])).toEqual([
{ path: 'main/data.iwd', size: 0, lastModified: 0 },
{ path: 'zone/map.ff', size: 10, lastModified: 100 }
]);
});
it.each([
[], [{ path: '../zone.ff', size: 1 }], [{ path: 'zone\\map.ff', size: 1 }],
[{ path: 'Zone/a', size: 1 }, { path: 'zone/A', size: 1 }],
[{ path: 'ZONE', size: 1 }, { path: 'zone/a', size: 1 }],
[{ path: 'zone/a', size: -1 }], [{ path: 'zone/a', size: 1.5 }],
[{ path: 'zone/a', size: Number.MAX_SAFE_INTEGER }, { path: 'zone/b', size: 1 }],
[{ path: 'zone/a', size: 1, lastModified: -1 }], [{ path: 'zone/a', size: 1, lastModified: 1.5 }],
[{ path: 'zone/a', size: 1, lastModified: null }]
].map(plan => ({ plan })))('rejects ambiguous or unsafe plan $plan', ({ plan }) => {
expect(() => validateUploadPlan(plan)).toThrow();
});
});
describe('bounded resumable file transfer', () => {
it('resumes arbitrary chunks by filesystem offset and produces a distribution-compatible manifest', async () => {
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 0, complete: false });
expect(await receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).toEqual({ receivedBytes: 4, complete: false });
// A new request discovers its checkpoint from disk; no process-local receipt is required.
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 4, complete: false });
expect(await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'))).toEqual({ receivedBytes: 10, complete: true });
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123456789');
const manifest = await withUploadLock(root, () => hashUploadedFiles(root, [file]));
expect(manifest).toEqual([{ path: file.path, size: 10, sha256: sha('0123456789') }]);
expect(manifestChecksum(manifest)).toBe(sha(JSON.stringify(manifest)));
await verifyGameFile(root, manifest[0]);
expect(await readdir(root)).toEqual(['zone']);
expect(await readdir(`${root}.chunks`)).toEqual([]);
});
it('reports committed progress after reply loss instead of appending duplicate bytes', async () => {
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toMatchObject({
name: 'UploadOffsetError', receivedBytes: 4, status: 409
});
await expect(receiveUploadChunk(root, file, chunk('89', 8), stream('89'))).rejects.toBeInstanceOf(UploadOffsetError);
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'));
});
it('can resume a verified append interrupted by process termination at an unaligned byte', async () => {
await mkdir(join(root, 'zone/english'), { recursive: true });
await writeFile(join(root, file.path), '0123456');
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 7, complete: false });
await receiveUploadChunk(root, file, chunk('789', 7), stream('789'));
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123456789');
});
it('distinguishes an absent empty file and creates it using the empty digest', async () => {
const empty = { path: 'main/empty.dat', size: 0 };
expect(await getUploadFileProgress(root, empty)).toEqual({ receivedBytes: 0, complete: false });
await expect(hashUploadedFiles(root, [empty])).rejects.toThrow();
expect(await receiveUploadChunk(root, empty, chunk(''), null)).toEqual({ receivedBytes: 0, complete: true });
expect(await getUploadFileProgress(root, empty)).toEqual({ receivedBytes: 0, complete: true });
expect(await hashUploadedFiles(root, [empty])).toEqual([{ ...empty, sha256: sha('') }]);
});
it('does not advance progress for wrong checksums, truncated requests, oversized streams or disconnects', async () => {
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
await expect(receiveUploadChunk(root, file, chunk('456789', 4), stream('xxxxxx'))).rejects.toThrow(/checksum/);
await expect(receiveUploadChunk(root, file, chunk('456789', 4), stream('45'))).rejects.toThrow(/size mismatch/);
await expect(receiveUploadChunk(root, file, chunk('456789', 4), stream('4567890'))).rejects.toThrow(/exceeds/);
let reads = 0;
const disconnect = new ReadableStream<Uint8Array>({ pull(controller) {
if (reads++ === 0) controller.enqueue(new TextEncoder().encode('45'));
else controller.error(new Error('Browser connection ended.'));
} });
await expect(receiveUploadChunk(root, file, chunk('456789', 4), disconnect)).rejects.toThrow(/connection ended/);
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 4, complete: false });
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
expect(await readdir(`${root}.chunks`)).toEqual([]);
await receiveUploadChunk(root, file, chunk('456789', 4), stream('456789'));
});
it('bounds request size before reading any body and rejects invalid metadata', async () => {
const tooLarge = MAX_UPLOAD_CHUNK_BYTES + 1;
await expect(receiveUploadChunk(root, { path: 'large.iwd', size: tooLarge }, {
offset: 0, size: tooLarge, sha256: sha('')
}, null)).rejects.toThrow(/Invalid upload chunk/);
for (const metadata of [
{ ...chunk('a'), offset: -1 }, { ...chunk('a'), offset: 0.5 }, { ...chunk('a'), size: 1.5 },
{ ...chunk('a'), sha256: 'wrong' }, chunk('', 0), chunk('01234567890', 0)
]) await expect(receiveUploadChunk(root, file, metadata, stream('a'))).rejects.toThrow(/Invalid upload chunk/);
await expect(receiveUploadChunk(root, file, chunk('a'), null)).rejects.toThrow(/body is required/);
});
it('streams a maximum-size chunk supplied in small pieces', async () => {
const piece = new Uint8Array(64 * 1024).fill(97);
let remaining = MAX_UPLOAD_CHUNK_BYTES;
const digest = createHash('sha256');
for (let i = 0; i < MAX_UPLOAD_CHUNK_BYTES / piece.length; i++) digest.update(piece);
const body = new ReadableStream<Uint8Array>({ pull(controller) {
if (remaining === 0) controller.close();
else { remaining -= piece.length; controller.enqueue(piece); }
} });
const large = { path: 'main/large.iwd', size: MAX_UPLOAD_CHUNK_BYTES };
const sha256 = digest.digest('hex');
await receiveUploadChunk(root, large, { offset: 0, size: large.size, sha256 }, body);
expect(await hashUploadedFiles(root, [large])).toEqual([{ ...large, sha256 }]);
});
it('rejects missing or incorrectly sized files before publication', async () => {
await receiveUploadChunk(root, file, chunk('0123'), stream('0123'));
await expect(hashUploadedFiles(root, [file])).rejects.toThrow(/Size verification/);
await writeFile(join(root, file.path), '01234567890');
await expect(getUploadFileProgress(root, file)).rejects.toThrow(/exceeds/);
});
it('refuses traversal and symlinks in destination and temporary storage', async () => {
await mkdir(root);
const outside = join(directory, 'outside');
await mkdir(outside);
await expect(receiveUploadChunk(root, { path: '../escape', size: 1 }, chunk('a'), stream('a'))).rejects.toThrow(/relative/);
await symlink(outside, join(root, 'zone'));
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toThrow(/non-regular/);
await rm(join(root, 'zone'));
await symlink(outside, `${root}.chunks`);
await expect(receiveUploadChunk(root, file, chunk('0123'), stream('0123'))).rejects.toThrow(/non-regular/);
expect(await readdir(outside)).toEqual([]);
});
});
describe('upload and publication coordination', () => {
it('serializes simultaneous writes with the same offset', async () => {
const responses = await Promise.allSettled([
receiveUploadChunk(root, file, chunk('0123'), stream('0123')),
receiveUploadChunk(root, file, chunk('0123'), stream('0123'))
]);
expect(responses.map(response => response.status)).toEqual(['fulfilled', 'rejected']);
expect((responses[1] as PromiseRejectedResult).reason).toMatchObject({ name: 'UploadOffsetError', receivedBytes: 4 });
expect(await readFile(join(root, file.path), 'utf8')).toBe('0123');
});
it('rechecks writable state inside the lock after queued publication and releases failed locks', async () => {
let finishPublication!: () => void;
const gate = new Promise<void>(resolve => { finishPublication = resolve; });
let published = false;
const publication = withUploadLock(root, async () => { await gate; published = true; });
const transfer = receiveUploadChunk(root, file, chunk('0123'), stream('0123'), () => {
if (published) throw new Error('Library is immutable.');
});
finishPublication();
await publication;
await expect(transfer).rejects.toThrow(/immutable/);
expect(await getUploadFileProgress(root, file)).toEqual({ receivedBytes: 0, complete: false });
await expect(withUploadLock(root, () => { throw new Error('Failed work'); })).rejects.toThrow(/Failed work/);
expect(await withUploadLock(root, () => 'next operation')).toBe('next operation');
});
});
+75
View File
@@ -0,0 +1,75 @@
import { describe, expect, it, vi } from 'vitest';
import { selectGameFolder } from '../../src/lib/ui/game-folder.js';
function selectedFile(path: string, size = 10) {
const read = vi.fn(() => { throw new Error('Folder selection must not read file bodies.'); });
const file = {
name: path.split('/').at(-1), webkitRelativePath: path, size, lastModified: 1_700_000_000_000,
arrayBuffer: read, text: read, stream: read, slice: read
} as unknown as File;
return { file, read };
}
describe('game folder selection', () => {
it('removes the selected root exactly once while preserving nested game paths', () => {
const first = selectedFile('T6/T6/zone/english/base.ff', 20);
const second = selectedFile('T6/main/iw_00.iwd', 30);
const result = selectGameFolder([first.file, second.file]);
expect(result.name).toBe('T6');
expect(result.size).toBe(50);
expect(result.files).toEqual([
{ path: 'T6/zone/english/base.ff', file: first.file },
{ path: 'main/iw_00.iwd', file: second.file }
]);
expect(first.read).not.toHaveBeenCalled();
expect(second.read).not.toHaveBeenCalled();
});
it('prepares multi-gigabyte and empty files using metadata alone', () => {
const large = selectedFile('Game/zone/base.ff', 7 * 1024 ** 3);
const empty = selectedFile('Game/empty.txt', 0);
const result = selectGameFolder([large.file, empty.file]);
expect(result.size).toBe(7 * 1024 ** 3);
expect(result.files.map(item => item.path)).toEqual(['empty.txt', 'zone/base.ff']);
expect(result.files[1].file).toBe(large.file);
expect(large.read).not.toHaveBeenCalled();
expect(empty.read).not.toHaveBeenCalled();
});
it('rejects case-insensitive duplicate file names', () => {
expect(() => selectGameFolder([
selectedFile('Game/Zone/base.ff').file,
selectedFile('Game/zone/BASE.ff').file
])).toThrow('duplicate game paths');
});
it('rejects a path used as both a file and a directory regardless of case', () => {
expect(() => selectGameFolder([
selectedFile('Game/zone/english/base.ff').file,
selectedFile('Game/ZONE').file
])).toThrow('same game path for a file and a directory');
});
it.each(['../base.ff', './base.ff', '/Game/base.ff', 'base.ff'])(
'rejects an absent or parent root in %s', path => {
expect(() => selectGameFolder([selectedFile(path).file])).toThrow('one game folder');
}
);
it.each(['Game/../base.ff', 'Game/zone/./base.ff', 'Game/zone//base.ff', 'Game/zone\\base.ff'])(
'rejects traversal and non-normalized content paths in %s', path => {
expect(() => selectGameFolder([selectedFile(path).file])).toThrow('unsupported game path');
}
);
it('rejects files from multiple selected roots', () => {
expect(() => selectGameFolder([
selectedFile('T5/main/base.ff').file,
selectedFile('T6/main/base.ff').file
])).toThrow('one game folder');
});
it('rejects empty selections', () => {
expect(() => selectGameFolder([])).toThrow('contains no files');
});
});
+263
View File
@@ -0,0 +1,263 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import Database from 'better-sqlite3';
import { readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const inventory = { hostname: 'games-1', protocolVersion: 1, capabilities: ['host.decommission'], dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3 };
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { vi.unstubAllEnvs(); await fixture.cleanup(); });
async function enrolled() {
const hosts = await import('../../src/lib/server/hosts/index.js');
const host = hosts.createHost({ name: 'Game host' });
const credentials = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
return { ...hosts, host, credentials };
}
async function finishCleanup(hosts: Awaited<ReturnType<typeof enrolled>>) {
const result = hosts.deleteHost(hosts.host.id);
const operation = hosts.getHostOperation(result.operationId!)!;
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
const token = operation.payload.receiptToken as string;
hosts.receiveDecommissionReceipt(operation.id, token, { stage: 'cleaned' });
hosts.receiveDecommissionReceipt(operation.id, token, { stage: 'completed' });
return operation;
}
async function instance(hostId: string, retired = false) {
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at,retired_at)
VALUES('instance','t5_zombie_theater_1',?,'library','plutainer:test',28960,?,?)`)
.run(hostId, new Date().toISOString(), retired ? new Date().toISOString() : null);
}
describe('host deletion', () => {
it('adds the deletion marker to existing host schemas without losing hosts', async () => {
const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js');
const legacy = new Database(':memory:');
try {
legacy.exec("CREATE TABLE ec_hosts(id TEXT PRIMARY KEY, name TEXT NOT NULL); INSERT INTO ec_hosts VALUES('old','Existing host')");
ensureHostSchema(legacy); ensureHostSchema(legacy);
expect(legacy.prepare('SELECT * FROM ec_hosts').get()).toEqual({ id: 'old', name: 'Existing host', deleted_at: null });
} finally { legacy.close(); }
});
it('deletes an unenrolled host and invalidates its unused enrollment', async () => {
const hosts = await import('../../src/lib/server/hosts/index.js');
const host = hosts.createHost({ name: 'Not installed' });
const enrollment = hosts.createEnrollment(host.id);
expect(hosts.hostDeletionStatus(host.id)).toEqual({ allowed: true, blockers: [] });
hosts.deleteHost(host.id);
expect(hosts.getHost(host.id)).toBeNull(); expect(hosts.listHosts()).toEqual([]);
expect(() => hosts.enrollHost(enrollment.token, inventory)).toThrow('invalid or expired');
expect(() => hosts.createEnrollment(host.id)).toThrow('not found');
expect(() => hosts.deleteHost(host.id)).toThrow(expect.objectContaining({ status: 404 }));
});
it('waits for host cleanup before revoking credentials and retains controller libraries and history', async () => {
const hosts = await enrolled();
const other = hosts.createHost({ name: 'Keep' });
const otherCredentials = hosts.enrollHost(hosts.createEnrollment(other.id).token, inventory);
const enrollment = hosts.createEnrollment(hosts.host.id);
const completed = hosts.queueHostOperation(hosts.host.id, 'host.inspect', {});
hosts.reportHostOperation(hosts.host.id, completed.id, { state: 'completed', result: { hostname: 'games-1' } });
await instance(hosts.host.id, true);
const file = join(fixture.root, 'game-files-sentinel');
await writeFile(file, 'Game files remain');
const { db } = await import('../../src/lib/server/db/index.js');
const { ensureGameFileSchema } = await import('../../src/lib/server/game-files/schema.js');
ensureGameFileSchema(db());
db().prepare("INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES('library','T5','t5','',?)").run(new Date().toISOString());
db().prepare(`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,storage_location,source_host_id,source_volume)
VALUES('version','library','1','','[]','ready',?,?,?,'t5_game_files')`).run(new Date().toISOString(), file, hosts.host.id);
db().prepare("INSERT INTO host_library_caches(host_id,version_id,status,updated_at) VALUES(?,'version','ready',?)").run(hosts.host.id, new Date().toISOString());
const queued = hosts.deleteHost(hosts.host.id);
expect(queued.deleted).toBe(false);
expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('queued');
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
await finishCleanup(hosts);
expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull();
expect(hosts.authenticateCredential(otherCredentials.credential)?.id).toBe(other.id);
expect(() => hosts.enrollHost(enrollment.token, inventory)).toThrow('invalid or expired');
expect(hosts.listHosts().map(host => host.id)).toEqual([other.id]);
expect(hosts.getHostOperation(completed.id)).toMatchObject({ state: 'completed', result: { hostname: 'games-1' } });
expect(hosts.listHostOperations(hosts.host.id)).toHaveLength(2);
expect(db().prepare('SELECT mode,deleted_at FROM ec_hosts WHERE id=?').get(hosts.host.id)).toEqual({ mode: 'disabled', deleted_at: expect.any(String) });
expect(db().prepare('SELECT revoked_at FROM ec_host_credentials WHERE host_id=?').get(hosts.host.id)).toEqual({ revoked_at: expect.any(String) });
expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({ count: 1 });
expect(db().prepare('SELECT COUNT(*) AS count FROM host_library_caches').get()).toEqual({ count: 1 });
expect(db().prepare("SELECT state,source_host_id FROM game_file_versions WHERE id='version'").get()).toEqual({ state: 'ready', source_host_id: hosts.host.id });
expect(await readFile(file, 'utf8')).toBe('Game files remain');
expect(() => hosts.heartbeat(hosts.host.id, inventory)).toThrow('not found');
expect(() => hosts.rotateCredential(hosts.host.id)).toThrow('not found');
expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow('not found');
expect(() => hosts.reportHostOperation(hosts.host.id, completed.id, { state: 'running' })).toThrow('not found');
expect(() => hosts.updateHost(hosts.host.id, { mode: 'active' })).toThrow('not found');
});
it('cancels only undelivered queued operations and retains their journal', async () => {
const hosts = await enrolled();
const operations = ['runtime.pull', 'library.download', 'credentials.rotate'].map(kind => hosts.queueHostOperation(hosts.host.id, kind, {}));
expect(hosts.hostDeletionStatus(hosts.host.id).allowed).toBe(true);
hosts.deleteHost(hosts.host.id);
const { db } = await import('../../src/lib/server/db/index.js');
for (const operation of operations) {
expect(hosts.getHostOperation(operation.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('before this operation was delivered') });
expect(db().prepare('SELECT state FROM ec_host_operation_events WHERE operation_id=? ORDER BY id').all(operation.id)).toEqual([{ state: 'pending' }, { state: 'failed' }]);
}
});
it('blocks live instances even when no server record or port reservation exists', async () => {
const hosts = await enrolled();
await instance(hosts.host.id);
const status = hosts.hostDeletionStatus(hosts.host.id);
expect(status.allowed).toBe(false); expect(status.blockers.join(' ')).toContain('t5_zombie_theater_1');
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
});
it('blocks residual port reservations independently of live instances', async () => {
const hosts = await enrolled();
hosts.reserveHostPort(hosts.host.id, 'unresolved-instance');
expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('reserved game port')] });
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
hosts.releaseHostPorts('unresolved-instance');
await finishCleanup(hosts);
expect(hosts.getHost(hosts.host.id)).toBeNull();
});
it.each(['pending', 'accepted', 'running', 'unknown'] as const)('blocks unresolved %s work after delivery without revoking access or cancelling other work', async state => {
const hosts = await enrolled();
const operation = hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {});
hosts.pendingHostOperations(hosts.host.id);
if (state !== 'pending') hosts.reportHostOperation(hosts.host.id, operation.id, { state });
const queued = hosts.queueHostOperation(hosts.host.id, 'library.download', {});
hosts.updateHost(hosts.host.id, { mode: 'disabled' });
expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('delivered operation')] });
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
expect(hosts.getHostOperation(queued.id)?.state).toBe('pending');
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' });
hosts.deleteHost(hosts.host.id);
expect(hosts.getHostOperation(queued.id)?.state).toBe('failed');
});
it('rechecks blockers on DELETE even after a successful eligibility check', async () => {
const hosts = await enrolled();
const route = await import('../../src/routes/api/hosts/[id]/+server.js');
const url = `https://ec.test/api/hosts/${hosts.host.id}`;
const params = { id: hosts.host.id };
expect(await route.GET({ request: new Request(url), params }).json()).toMatchObject({ deletion: { allowed: true, blockers: [] } });
const operation = hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {});
hosts.pendingHostOperations(hosts.host.id);
const blocked = route.DELETE({ request: new Request(url, { method: 'DELETE' }), params });
expect(blocked.status).toBe(409);
expect(await blocked.json()).toMatchObject({ error: expect.any(String), blockers: [expect.stringContaining('delivered operation')] });
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' });
const removed = route.DELETE({ request: new Request(url, { method: 'DELETE' }), params });
expect(removed.status).toBe(202); expect(await removed.json()).toMatchObject({ deleted: false, operationId: expect.any(String) });
await finishCleanup(hosts);
expect(route.DELETE({ request: new Request(url, { method: 'DELETE' }), params }).status).toBe(404);
});
it('keeps DELETE restricted to admin sessions and same-origin management requests', async () => {
const hosts = await enrolled();
const { routePermission, roleAllows } = await import('../../src/lib/server/auth/access.js');
const permission = routePermission('/api/hosts/[id]', 'DELETE');
expect(roleAllows('admin', permission)).toBe(true);
expect(roleAllows('operator', permission)).toBe(false); expect(roleAllows('viewer', permission)).toBe(false);
const { DELETE } = await import('../../src/routes/api/hosts/[id]/+server.js');
for (const headers of [new Headers({ origin: 'https://attacker.test' }), new Headers({ authorization: `Bearer ${hosts.credentials.credential}` })]) {
expect(() => DELETE({ request: new Request('https://ec.test/api/hosts/id', { method: 'DELETE', headers }), params: { id: hosts.host.id } })).toThrow(expect.objectContaining({ status: 403 }));
}
expect(hosts.getHost(hosts.host.id)).not.toBeNull();
});
it('requires an uninstall-capable agent and preserves enrollment access for upgrades', async () => {
const hosts = await enrolled();
hosts.heartbeat(hosts.host.id, { ...inventory, capabilities: [] });
expect(hosts.hostDeletionStatus(hosts.host.id)).toMatchObject({ allowed: false, blockers: [expect.stringContaining('Update this host agent')] });
expect(() => hosts.deleteHost(hosts.host.id)).toThrow(expect.objectContaining({ status: 409 }));
expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy();
expect(hosts.getHost(hosts.host.id)?.mode).toBe('active');
});
it('keeps offline cleanup pending and blocks settings, new work, enrollment and credential revocation', async () => {
const hosts = await enrolled();
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2020-01-01T00:00:00Z', hosts.host.id);
const queued = hosts.deleteHost(hosts.host.id);
expect(hosts.deleteHost(hosts.host.id)).toEqual(queued);
expect(hosts.getHost(hosts.host.id)).toMatchObject({ connected: false, readiness: 'deleting', decommission: { stage: 'queued' } });
expect(hosts.pendingHostOperations(hosts.host.id)).toEqual([]);
expect(() => hosts.updateHost(hosts.host.id, { mode: 'disabled' })).toThrow('cleanup');
expect(() => hosts.queueHostOperation(hosts.host.id, 'runtime.pull', {})).toThrow('cleanup');
expect(() => hosts.reserveHostPort(hosts.host.id, 'late-instance')).toThrow('cleanup');
expect(hosts.hostPortAvailable(hosts.host.id, hosts.host.portMin)).toBe(false);
expect(() => hosts.createEnrollment(hosts.host.id)).toThrow('cleanup');
expect(() => hosts.revokeCredentials(hosts.host.id)).toThrow('cleanup');
hosts.heartbeat(hosts.host.id, inventory);
expect(hosts.pendingHostOperations(hosts.host.id).map(operation => operation.id)).toEqual([queued.operationId]);
});
it('requires scoped receipts in order and never treats an ordinary completion report or loss of contact as removal', async () => {
const hosts = await enrolled();
const queued = hosts.deleteHost(hosts.host.id);
const operation = hosts.getHostOperation(queued.operationId!)!;
const token = operation.payload.receiptToken as string;
const receipt = (stage: string, credential = token) => hosts.receiveDecommissionReceipt(operation.id, credential, { stage });
expect(() => receipt('cleaned')).toThrow(expect.objectContaining({ status: 409 }));
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
expect(() => receipt('cleaned', hosts.credentials.credential)).toThrow(expect.objectContaining({ status: 401 }));
expect(() => receipt('completed')).toThrow(expect.objectContaining({ status: 409 }));
expect(() => hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'completed' })).toThrow('cleanup worker');
receipt('cleaned');
expect(hosts.getHost(hosts.host.id)?.decommission?.stage).toBe('cleaned');
expect(hosts.authenticateCredential(hosts.credentials.credential)?.id).toBe(hosts.host.id);
const route = await import('../../src/routes/api/hosts/[id]/+server.js');
const detail = await route.GET({ request: new Request('https://ec.test/api/hosts/' + hosts.host.id), params: { id: hosts.host.id } }).text();
expect(detail).not.toContain(token);
receipt('completed');
expect(hosts.getHost(hosts.host.id)).toBeNull();
expect(receipt('completed')).toEqual({ acknowledged: true });
expect(receipt('cleaned')).toEqual({ acknowledged: true });
expect(hosts.getHostOperation(operation.id)?.state).toBe('completed');
expect(hosts.authenticateCredential(hosts.credentials.credential)).toBeNull();
});
it('retains cleanup failures and retries only work rejected before mutation with a new scoped receipt', async () => {
const hosts = await enrolled();
const first = hosts.deleteHost(hosts.host.id);
hosts.reportHostOperation(hosts.host.id, first.operationId!, { state: 'failed', error: 'Ownership marker requires repair' });
expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ stage: 'failed', retryable: true });
expect(hosts.createEnrollment(hosts.host.id).token).toBeTruthy();
const second = hosts.deleteHost(hosts.host.id);
expect(second.operationId).not.toBe(first.operationId);
const operation = hosts.getHostOperation(second.operationId!)!;
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
hosts.receiveDecommissionReceipt(operation.id, operation.payload.receiptToken as string, { stage: 'failed', error: 'Filesystem cleanup failed; retrying' });
expect(hosts.getHost(hosts.host.id)?.decommission).toMatchObject({ stage: 'failed', retryable: false, error: 'Filesystem cleanup failed; retrying' });
expect(hosts.deleteHost(hosts.host.id)).toEqual(second);
await finishCleanup(hosts);
expect(hosts.getHostOperation(first.operationId!)?.state).toBe('failed');
});
it('authenticates cleanup receipts independently after the agent credential has been revoked', async () => {
const hosts = await enrolled();
const queued = hosts.deleteHost(hosts.host.id);
const operation = hosts.getHostOperation(queued.operationId!)!;
hosts.reportHostOperation(hosts.host.id, operation.id, { state: 'running' });
const { POST } = await import('../../src/routes/api/agent/decommission/[id]/+server.js');
const send = (stage: string, token: string) => POST({ request: new Request(`https://ec.test/api/agent/decommission/${operation.id}`, {
method: 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, body: JSON.stringify({ stage })
}), params: { id: operation.id } });
expect((await send('cleaned', hosts.credentials.credential)).status).toBe(401);
expect((await send('cleaned', operation.payload.receiptToken as string)).status).toBe(200);
expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(200);
expect((await send('completed', operation.payload.receiptToken as string)).status).toBe(200);
});
});
+35 -1
View File
@@ -71,7 +71,7 @@ describe('native host readiness and reservations', () => {
expect(api.requireReadyHost(api.host.id).readiness).toBe('ready');
expect(() => api.requireReadyHost(api.host.id, { image: 'missing:image' })).toThrow('runtime');
expect(() => api.requireReadyHost(api.host.id, { libraryId: 'missing' })).toThrow('library');
expect(api.heartbeat(api.host.id, { ...inventory, libraries: [] }).readiness).toBe('preparing');
expect(api.heartbeat(api.host.id, { ...inventory, libraries: [] }).readiness).toBe('needs-game-files');
const host = api.heartbeat(api.host.id, { ...inventory, dockerRunning: false });
expect(host.connected).toBe(true); expect(host.readiness).toBe('degraded');
expect(() => api.requireReadyHost(host.id)).toThrow('Docker');
@@ -79,6 +79,40 @@ describe('native host readiness and reservations', () => {
vi.useFakeTimers(); vi.setSystemTime(Date.now() + 91_000);
expect(api.getHost(host.id)?.connectionState).toBe('offline'); expect(() => api.requireReadyHost(host.id)).toThrow('stale');
});
it.each([
{ missing: { libraries: [] }, expected: 'needs-game-files' },
{ missing: { runtimeImages: [] }, expected: 'needs-runtime' },
{ missing: { runtimeImages: [], libraries: [] }, expected: 'needs-runtime' }
])('names missing content as $expected until a prepared heartbeat arrives', async ({ missing, expected }) => {
const api = await enrolled();
const host = api.heartbeat(api.host.id, { ...inventory, ...missing });
expect(host.connected).toBe(true);
expect(host.readiness).toBe(expected);
expect(() => api.requireReadyHost(host.id)).toThrow();
expect(api.heartbeat(host.id, inventory).readiness).toBe('ready');
expect(api.requireReadyHost(host.id).id).toBe(host.id);
});
it('keeps setup work deliverable while new instance creation waits for missing game files', async () => {
const api = await enrolled();
const create = api.queueHostOperation(api.host.id, 'instance.create', {});
api.heartbeat(api.host.id, { ...inventory, libraries: [] });
expect(() => api.queueHostOperation(api.host.id, 'instance.create', {})).toThrow('game-file library');
const download = api.queueHostOperation(api.host.id, 'library.download', {});
const runtime = api.queueHostOperation(api.host.id, 'runtime.pull', {});
expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toEqual([download.id, runtime.id]);
api.heartbeat(api.host.id, { ...inventory, runtimeImages: [], libraries: [] });
expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toEqual([download.id, runtime.id]);
api.heartbeat(api.host.id, inventory);
expect(api.pendingHostOperations(api.host.id).map(operation => operation.id)).toContain(create.id);
});
it('gives enrollment, infrastructure failure and host modes priority over missing content', async () => {
const api = await enrolled();
expect(api.createHost({ name: 'New host' }).readiness).toBe('preparing');
expect(api.heartbeat(api.host.id, { ...inventory, libraries: [], dockerRunning: false }).readiness).toBe('degraded');
api.heartbeat(api.host.id, { ...inventory, libraries: [] });
expect(api.updateHost(api.host.id, { mode: 'draining' }).readiness).toBe('draining');
expect(api.updateHost(api.host.id, { mode: 'disabled' }).readiness).toBe('disabled');
});
it('blocks new deployments on a draining or disabled host without altering existing operations', async () => {
const api = await enrolled(); const existing = api.queueHostOperation(api.host.id, 'instance.inspect', {}, { instanceId: 'immutable-id', generation: 1 });
api.updateHost(api.host.id, { mode: 'draining' });