josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Provision instance bank credentials automatically (Refs #146)

Browse Source
This commit is contained in:
josh
2026-10-12 20:44:01 -04:00
parent e071e5b574
commit d14c6a7ff4
21 changed files with 634 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
static/agent/ec-agent.py
+96 -6
View File
@@ -35,7 +35,7 @@ import urllib.parse
import urllib.request
import uuid
VERSION = "1.8.7"
VERSION = "1.9.0"
LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024
MANIFEST_MAX_ENTRIES = 10000
@@ -1503,7 +1503,7 @@ class Agent:
if line.startswith("PRETTY_NAME="):
os_name = line.split("=", 1)[1].strip('"')
update = self.update_inventory()
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
plutonium_cache = self.plutonium_cache_inventory()
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
@@ -1657,7 +1657,7 @@ class Agent:
# record. Whenever preparation was recorded, enforce that exact image.
if image in prepared and prepared[image] != image_id:
raise AgentError("Prepared runtime image has changed. An admin must prepare it again before launching.")
files = payload.get("files", [])
files = self.initial_files(payload)
if not isinstance(files, list) or len(files) > 10000:
raise Rejected("Invalid initial file list")
total, paths = 0, set()
@@ -1911,7 +1911,7 @@ class Agent:
self.operation_phase = "seeding-runtime"
self.seed_plutonium(app, image)
self.operation_phase = "writing-create-files"
self.write_initial_files(app, payload.get("files", []))
self.write_initial_files(app, self.initial_files(payload))
self.operation_phase = "persisting-create-metadata"
atomic_json(directory / "spec.json", payload)
env_path = directory / "environment"
@@ -1944,6 +1944,25 @@ class Agent:
if self.executing_id and self.executing_id in self.cancelled_creations:
raise AgentError("Creation cancelled; waiting for instance cleanup")
def initial_files(self, payload):
files = payload.get("files", [])
if not isinstance(files, list):
raise Rejected("Invalid initial file list")
bank = payload.get("bank")
if bank is None:
return files
if not isinstance(bank, dict) or not isinstance(bank.get("token"), str) or not re.fullmatch(r"ec_bank_[A-Za-z0-9_-]{32,128}", bank["token"]):
raise Rejected("Invalid instance bank credential")
path = bank.get("configPath")
scoped(self.root / "agent" / "validation", path)
if PurePosixPath(path).name != "electric_cherry_bank.cfg":
raise Rejected("Invalid private bank config path")
controller = self.config["controller"].rstrip("/")
if any(char in controller for char in ('"', "\\", "\n", "\r", "\x00")):
raise Rejected("Invalid bank controller URL")
content = f'set tws_bank_token "{bank["token"]}"\nset tws_bank_url "{controller}/api/bank"\n'.encode()
return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}]
def write_initial_files(self, root, files):
if not isinstance(files, list) or len(files) > 10000:
raise AgentError("Invalid file list")
@@ -1992,6 +2011,72 @@ class Agent:
if (time.monotonic() - started) * 1000 >= ttl:
raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation")
def settle_bank(self, instance_id, generation, operation_id=None):
"""Drain stopped runtime journals before discarding any generation's storage.
Use only our configured controller and host credential. Never follow an
endpoint or symlink supplied by a game journal. Lost replies are retried
with the original request identity; files remain until teardown succeeds.
"""
app = scoped(self.instance_dir(instance_id, generation), "app")
if not app.exists():
return
if self.inspect(instance_id, generation)["running"]:
raise AgentError("Bank settlement requires a stopped instance")
operation_id = operation_id or self.executing_id
visited = 0
deadline = time.monotonic() + 30
journals = []
def unreadable(_error):
raise AgentError("Bank journal discovery failed; runtime storage retained")
for directory, directories, files in os.walk(app, followlinks=False, onerror=unreadable):
relative = Path(directory).relative_to(app)
inside_bank = "tws-bank" in relative.parts
visited += 1
if visited > 100000 or time.monotonic() > deadline:
raise AgentError("Bank journal discovery limit exceeded; runtime storage retained")
for name in directories + files:
if (inside_bank or name == "tws-bank") and (Path(directory) / name).is_symlink():
raise AgentError("Bank journal paths must not be symlinks")
if not inside_bank:
continue
for name in files:
if name.endswith(".json"):
journals.append((relative / name).as_posix())
if len(journals) > 4096:
raise AgentError("Too many bank journals; runtime storage retained")
for path in sorted(journals):
try:
raw = managed_read(app, path)
except Rejected:
raise AgentError("Bank journal cannot be read; runtime storage retained") from None
if raw is None or len(raw) > 16 * 1024 * 1024:
raise AgentError("Bank journal is unavailable or oversized")
try:
journal = json.loads(raw)
reports = journal["reports"]
if journal["endpoint"].rstrip("/") != self.config["controller"].rstrip("/") + "/api/bank":
raise AgentError("Bank journal belongs to a different controller; runtime storage retained")
if type(journal["version"]) is not int or journal["version"] != 1 or not isinstance(reports, list) or len(reports) > 4096:
raise ValueError()
# Controller independently binds this operation to the instance/server.
for report in reports:
if not isinstance(report, str) or len(report.encode()) > 20000:
raise ValueError()
transfer = json.loads(report)
action = transfer["decision"]
if action not in ("commit", "cancel") or transfer["server"] != journal["server"] or not operation_id:
raise ValueError()
response = self.request("POST", "/api/agent/bank/settle", {
"operationId": operation_id, "action": action, "transfer": transfer})
expected = ("committed",) if action == "commit" else ("cancelled", "rejected")
if not isinstance(response, dict) or response.get("request_id") != transfer["request_id"] or response.get("state") not in expected:
raise AgentError("Bank settlement was not acknowledged; runtime storage retained")
except (ValueError, TypeError, KeyError, AttributeError):
raise AgentError("Bank journal is invalid; runtime storage retained") from None
def lifecycle(self, kind, instance_id, generation, payload):
unhealthy_recovery = payload.get("unhealthyRecovery") is True
if unhealthy_recovery and (kind not in ("instance.restart", "instance.rebuild") or payload.get("stageId") or "autoscaling" in payload):
@@ -2043,6 +2128,7 @@ class Agent:
if health["exists"]:
self.docker.call("container", "stop", "--time", "30", name)
self.docker.call("container", "rm", name)
self.settle_bank(instance_id, generation)
if kind == "instance.rebuild":
result = self.create(instance_id, next_generation, spec)
self.operation_phase = "removing-old-instance"
@@ -2062,7 +2148,7 @@ class Agent:
raise AgentError("Unsupported instance operation")
return {**self.inspect(instance_id, generation), "before": before}
def cancel_creation(self, instance_id, generation, payload):
def cancel_creation(self, instance_id, generation, payload, operation_id=None):
"""Explicit launch cancellation, fenced to its immutable creation identity.
The operation loop serializes this with create. Once delivered, removal
@@ -2106,6 +2192,7 @@ class Agent:
self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, target_generation))
if self.inspect(instance_id, target_generation)["exists"]:
raise AgentError("Cancelled creation container still exists")
self.settle_bank(instance_id, target_generation, operation_id)
directory = self.instance_dir(instance_id, target_generation)
if directory.exists():
shutil.rmtree(directory)
@@ -2747,6 +2834,7 @@ class Agent:
if not unchanged_generation() or not both_missing():
return row
self.settle_bank(instance_id, generation, operation["id"])
# Preparation can leave runtime/configuration files before create's
# checkpoint. A fresh operation must not inherit those partial files.
expected_spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
@@ -2793,7 +2881,7 @@ class Agent:
if kind == "instance.remove" and "autoscaling" in payload and not checkpoint:
return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision")
if kind == "instance.cancel-create":
return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload))
return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload, operation["id"]))
if kind == "instance.create":
marker = self.instance_dir(instance_id, generation).parent / "current.json"
if marker.exists():
@@ -2885,6 +2973,7 @@ class Agent:
directory = self.instance_dir(instance_id, observed_generation)
atomic_json(directory.parent / "current.json", {"generation": observed_generation})
if kind == "instance.rebuild" and not self.inspect(instance_id, generation)["exists"]:
self.settle_bank(instance_id, generation, operation["id"])
old = self.instance_dir(instance_id, generation)
if old.exists():
shutil.rmtree(old)
@@ -2899,6 +2988,7 @@ class Agent:
elif kind == "instance.remove":
proven = not health["exists"]
if proven:
self.settle_bank(instance_id, generation, operation["id"])
directory = self.instance_dir(instance_id, generation)
if directory.exists():
shutil.rmtree(directory)