From d14c6a7ff48e013b4ebe75b81c77424d50f188f5 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 12 Oct 2026 20:44:01 -0400 Subject: [PATCH] Provision instance bank credentials automatically (Refs #146) --- .env.example | 4 +- agent/ec_agent.py | 102 ++++++++++++++- agent/tests/test_agent.py | 103 +++++++++++++++ docker-compose.yml | 1 - docs/guides/bank.md | 75 +++++++---- docs/guides/native-hosts.md | 9 ++ src/lib/server/auth/bank.ts | 71 +++++----- src/lib/server/db/bank-credentials.ts | 22 ++++ src/lib/server/db/migrations.ts | 4 +- src/lib/server/fleet/bank.ts | 21 ++- .../server/provisioning/launch-preparation.ts | 1 + src/lib/server/provisioning/launch-worker.ts | 16 ++- src/lib/server/provisioning/native.ts | 2 + src/routes/api/agent/bank/settle/+server.ts | 28 ++++ src/routes/api/bank/[action]/+server.ts | 3 +- static/agent/ec-agent.py | 102 ++++++++++++++- static/agent/manifest.json | 4 +- static/agent/manifest.json.sig | 8 -- tests/unit/bank.test.ts | 122 +++++++++++++++--- tests/unit/native-launch-preparation.test.ts | 44 +++++++ tests/unit/runtime-config.test.ts | 1 + 21 files changed, 634 insertions(+), 109 deletions(-) create mode 100644 src/lib/server/db/bank-credentials.ts create mode 100644 src/routes/api/agent/bank/settle/+server.ts delete mode 100644 static/agent/manifest.json.sig diff --git a/.env.example b/.env.example index 66d8c4c..3184ea1 100644 --- a/.env.example +++ b/.env.example @@ -7,9 +7,7 @@ EC_SETUP_TOKEN= # Game reporters send Authorization: Bearer or X-EC-Telemetry-Token. # Telemetry ingestion is disabled until a token of at least 32 chars is set. EC_TELEMETRY_TOKEN= -# Unique per-server bank secrets: JSON object mapping exact server names to -# ec_bank_<32+ random URL-safe characters>. Never reuse the telemetry secret. -EC_BANK_TOKENS={} +# Bank credentials are generated and injected automatically per native instance. EC_BANK_MAX_BALANCE=250000 # Persistent SQLite metadata, game-file storage, and beta archives. # For an existing deployment, retain its original data directory. diff --git a/agent/ec_agent.py b/agent/ec_agent.py index 90d8bb0..be5dc3f 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -35,7 +35,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.8.7" +VERSION = "1.9.0" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -1503,7 +1503,7 @@ class Agent: if line.startswith("PRETTY_NAME="): os_name = line.split("=", 1)[1].strip('"') update = self.update_inventory() - capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) + capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) plutonium_cache = self.plutonium_cache_inventory() return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} @@ -1657,7 +1657,7 @@ class Agent: # record. Whenever preparation was recorded, enforce that exact image. if image in prepared and prepared[image] != image_id: raise AgentError("Prepared runtime image has changed. An admin must prepare it again before launching.") - files = payload.get("files", []) + files = self.initial_files(payload) if not isinstance(files, list) or len(files) > 10000: raise Rejected("Invalid initial file list") total, paths = 0, set() @@ -1911,7 +1911,7 @@ class Agent: self.operation_phase = "seeding-runtime" self.seed_plutonium(app, image) self.operation_phase = "writing-create-files" - self.write_initial_files(app, payload.get("files", [])) + self.write_initial_files(app, self.initial_files(payload)) self.operation_phase = "persisting-create-metadata" atomic_json(directory / "spec.json", payload) env_path = directory / "environment" @@ -1944,6 +1944,25 @@ class Agent: if self.executing_id and self.executing_id in self.cancelled_creations: raise AgentError("Creation cancelled; waiting for instance cleanup") + def initial_files(self, payload): + files = payload.get("files", []) + if not isinstance(files, list): + raise Rejected("Invalid initial file list") + bank = payload.get("bank") + if bank is None: + return files + if not isinstance(bank, dict) or not isinstance(bank.get("token"), str) or not re.fullmatch(r"ec_bank_[A-Za-z0-9_-]{32,128}", bank["token"]): + raise Rejected("Invalid instance bank credential") + path = bank.get("configPath") + scoped(self.root / "agent" / "validation", path) + if PurePosixPath(path).name != "electric_cherry_bank.cfg": + raise Rejected("Invalid private bank config path") + controller = self.config["controller"].rstrip("/") + if any(char in controller for char in ('"', "\\", "\n", "\r", "\x00")): + raise Rejected("Invalid bank controller URL") + content = f'set tws_bank_token "{bank["token"]}"\nset tws_bank_url "{controller}/api/bank"\n'.encode() + return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}] + def write_initial_files(self, root, files): if not isinstance(files, list) or len(files) > 10000: raise AgentError("Invalid file list") @@ -1992,6 +2011,72 @@ class Agent: if (time.monotonic() - started) * 1000 >= ttl: raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation") + def settle_bank(self, instance_id, generation, operation_id=None): + """Drain stopped runtime journals before discarding any generation's storage. + + Use only our configured controller and host credential. Never follow an + endpoint or symlink supplied by a game journal. Lost replies are retried + with the original request identity; files remain until teardown succeeds. + """ + app = scoped(self.instance_dir(instance_id, generation), "app") + if not app.exists(): + return + if self.inspect(instance_id, generation)["running"]: + raise AgentError("Bank settlement requires a stopped instance") + operation_id = operation_id or self.executing_id + visited = 0 + deadline = time.monotonic() + 30 + journals = [] + + def unreadable(_error): + raise AgentError("Bank journal discovery failed; runtime storage retained") + + for directory, directories, files in os.walk(app, followlinks=False, onerror=unreadable): + relative = Path(directory).relative_to(app) + inside_bank = "tws-bank" in relative.parts + visited += 1 + if visited > 100000 or time.monotonic() > deadline: + raise AgentError("Bank journal discovery limit exceeded; runtime storage retained") + for name in directories + files: + if (inside_bank or name == "tws-bank") and (Path(directory) / name).is_symlink(): + raise AgentError("Bank journal paths must not be symlinks") + if not inside_bank: + continue + for name in files: + if name.endswith(".json"): + journals.append((relative / name).as_posix()) + if len(journals) > 4096: + raise AgentError("Too many bank journals; runtime storage retained") + for path in sorted(journals): + try: + raw = managed_read(app, path) + except Rejected: + raise AgentError("Bank journal cannot be read; runtime storage retained") from None + if raw is None or len(raw) > 16 * 1024 * 1024: + raise AgentError("Bank journal is unavailable or oversized") + try: + journal = json.loads(raw) + reports = journal["reports"] + if journal["endpoint"].rstrip("/") != self.config["controller"].rstrip("/") + "/api/bank": + raise AgentError("Bank journal belongs to a different controller; runtime storage retained") + if type(journal["version"]) is not int or journal["version"] != 1 or not isinstance(reports, list) or len(reports) > 4096: + raise ValueError() + # Controller independently binds this operation to the instance/server. + for report in reports: + if not isinstance(report, str) or len(report.encode()) > 20000: + raise ValueError() + transfer = json.loads(report) + action = transfer["decision"] + if action not in ("commit", "cancel") or transfer["server"] != journal["server"] or not operation_id: + raise ValueError() + response = self.request("POST", "/api/agent/bank/settle", { + "operationId": operation_id, "action": action, "transfer": transfer}) + expected = ("committed",) if action == "commit" else ("cancelled", "rejected") + if not isinstance(response, dict) or response.get("request_id") != transfer["request_id"] or response.get("state") not in expected: + raise AgentError("Bank settlement was not acknowledged; runtime storage retained") + except (ValueError, TypeError, KeyError, AttributeError): + raise AgentError("Bank journal is invalid; runtime storage retained") from None + def lifecycle(self, kind, instance_id, generation, payload): unhealthy_recovery = payload.get("unhealthyRecovery") is True if unhealthy_recovery and (kind not in ("instance.restart", "instance.rebuild") or payload.get("stageId") or "autoscaling" in payload): @@ -2043,6 +2128,7 @@ class Agent: if health["exists"]: self.docker.call("container", "stop", "--time", "30", name) self.docker.call("container", "rm", name) + self.settle_bank(instance_id, generation) if kind == "instance.rebuild": result = self.create(instance_id, next_generation, spec) self.operation_phase = "removing-old-instance" @@ -2062,7 +2148,7 @@ class Agent: raise AgentError("Unsupported instance operation") return {**self.inspect(instance_id, generation), "before": before} - def cancel_creation(self, instance_id, generation, payload): + def cancel_creation(self, instance_id, generation, payload, operation_id=None): """Explicit launch cancellation, fenced to its immutable creation identity. The operation loop serializes this with create. Once delivered, removal @@ -2106,6 +2192,7 @@ class Agent: self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, target_generation)) if self.inspect(instance_id, target_generation)["exists"]: raise AgentError("Cancelled creation container still exists") + self.settle_bank(instance_id, target_generation, operation_id) directory = self.instance_dir(instance_id, target_generation) if directory.exists(): shutil.rmtree(directory) @@ -2747,6 +2834,7 @@ class Agent: if not unchanged_generation() or not both_missing(): return row + self.settle_bank(instance_id, generation, operation["id"]) # Preparation can leave runtime/configuration files before create's # checkpoint. A fresh operation must not inherit those partial files. expected_spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")} @@ -2793,7 +2881,7 @@ class Agent: if kind == "instance.remove" and "autoscaling" in payload and not checkpoint: return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision") if kind == "instance.cancel-create": - return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload)) + return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload, operation["id"])) if kind == "instance.create": marker = self.instance_dir(instance_id, generation).parent / "current.json" if marker.exists(): @@ -2885,6 +2973,7 @@ class Agent: directory = self.instance_dir(instance_id, observed_generation) atomic_json(directory.parent / "current.json", {"generation": observed_generation}) if kind == "instance.rebuild" and not self.inspect(instance_id, generation)["exists"]: + self.settle_bank(instance_id, generation, operation["id"]) old = self.instance_dir(instance_id, generation) if old.exists(): shutil.rmtree(old) @@ -2899,6 +2988,7 @@ class Agent: elif kind == "instance.remove": proven = not health["exists"] if proven: + self.settle_bank(instance_id, generation, operation["id"]) directory = self.instance_dir(instance_id, generation) if directory.exists(): shutil.rmtree(directory) diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 1122f73..6dfce44 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -1644,6 +1644,109 @@ class AgentTests(unittest.TestCase): self.assertEqual(self.agent.execute(self.operation("instance.remove"))["state"], "failed") self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls)) + def test_create_materializes_private_bank_config_from_instance_credential(self): + secret = "ec_bank_" + "a" * 43 + payload = {"name": "server", "game": "t5", "libraryId": self.library_id, + "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960, + "bank": {"token": secret, "configPath": "configs/electric_cherry_bank.cfg"}} + operation = self.operation("instance.create", payload, "create") + self.assertEqual(self.agent.execute(operation)["state"], "completed") + path = self.agent.instance_dir(INSTANCE, 1) / "app/configs/electric_cherry_bank.cfg" + self.assertEqual(path.read_text(), f'set tws_bank_token "{secret}"\nset tws_bank_url "https://controller.example/api/bank"\n') + self.assertIn("bank.settlement", self.agent.inventory()["capabilities"]) + payload["bank"]["token"] = secret + '\nset bad "1"' + with self.assertRaises(AgentError): + self.agent.initial_files(payload) + + def bank_journal(self, decisions=("commit", "cancel"), game="t5"): + path = self.agent.instance_dir(INSTANCE, 1) / f"app/runtime/tws-bank/{game}/scope.json" + reports = [json.dumps({"server": "t5_zombie_test_1", "guid": "ace", "match": "match-one", "request_id": f"{game}-{i}", "operation": "deposit", "amount": 2000, "capacity": 1000000, "decision": decision}) for i, decision in enumerate(decisions)] + module.atomic_json(path, {"version": 1, "server": "t5_zombie_test_1", "endpoint": "https://controller.example/api/bank", "reports": reports}) + return path + + def bank_acknowledgment(self, method, path, payload): + self.assertEqual((method, path), ("POST", "/api/agent/bank/settle")) + self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"]) + return {"state": "committed" if payload["action"] == "commit" else "cancelled", "request_id": payload["transfer"]["request_id"]} + + def test_remove_settles_both_game_journals_after_stop_before_deleting_storage(self): + self.create() + paths = [self.bank_journal(game=game) for game in ("t5", "t6")] + def acknowledge(*args): + self.assertTrue(all(path.exists() for path in paths)) + self.assertEqual(args[2]["operationId"], "remove") + return self.bank_acknowledgment(*args) + with patch.object(self.agent, "request", side_effect=acknowledge) as request: + result = self.agent.execute(self.operation("instance.remove", {"safetyOverride": True}, "remove")) + self.assertEqual(result["state"], "completed", result) + self.assertEqual(request.call_count, 4) + self.assertTrue(all(not path.exists() for path in paths)) + + def test_remove_retains_bank_journal_during_outage_and_resumes_after_agent_restart(self): + self.create() + path = self.bank_journal() + operation = self.operation("instance.remove", {"safetyOverride": True}, "remove") + with patch.object(self.agent, "request", side_effect=AgentError("Controller request failed")): + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + self.assertTrue(path.exists()) + self.agent.journal.close() + self.agent = Agent(self.config, self.docker) + calls = list(self.docker.calls) + with patch.object(self.agent, "request", side_effect=self.bank_acknowledgment): + self.assertEqual(self.agent.execute(operation)["state"], "completed") + self.assertFalse(path.exists()) + self.assertEqual(self.docker.calls, calls) + + def test_rebuild_settles_before_starting_replacement_and_retains_unknown_decisions(self): + payload = self.create() + path = self.bank_journal(("commit",)) + operation = self.operation("instance.rebuild", {"spec": payload, "nextGeneration": 2, "safetyOverride": True}, "rebuild") + with patch.object(self.agent, "request", return_value={"state": "committed", "request_id": "wrong"}): + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + self.assertTrue(path.exists()) + self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"]) + with patch.object(self.agent, "request", side_effect=self.bank_acknowledgment): + # Existing rebuild recovery records the interrupted create as failed; + # a fresh operation can then rebuild without replaying the old command. + self.assertEqual(self.agent.execute(operation)["state"], "failed") + retry = self.operation("instance.rebuild", operation["payload"], "replacement") + self.assertEqual(self.agent.execute(retry)["state"], "completed") + self.assertFalse(path.exists()) + self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"]) + + def test_cancel_creation_retries_bank_settlement_before_deleting_runtime(self): + self.create() + path = self.bank_journal(("cancel",)) + operation = self.operation("instance.cancel-create", {"createOperationId": "create"}, "cancel") + with patch.object(self.agent, "request", side_effect=AgentError("offline")): + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + self.assertTrue(path.exists()) + with patch.object(self.agent, "request", side_effect=self.bank_acknowledgment): + self.assertEqual(self.agent.execute(operation)["state"], "completed") + self.assertFalse(path.exists()) + + def test_malformed_or_symlinked_bank_journal_blocks_storage_deletion(self): + self.create() + path = self.bank_journal() + path.write_text('{broken') + operation = self.operation("instance.remove", {"safetyOverride": True}, "remove") + with patch.object(self.agent, "request") as request: + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + self.assertTrue(path.exists()) + outside = Path(self.temp.name) / "outside.json" + outside.write_text('{"reports": []}') + self.bank_journal() + journal = json.loads(path.read_text()) + journal["endpoint"] = "https://untrusted.example/api/bank" + path.write_text(json.dumps(journal)) + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + path.unlink() + path.symlink_to(outside) + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + self.assertTrue(path.is_symlink()) + request.assert_not_called() + def test_rebuild_moves_generation_and_rejects_old_commands(self): payload = self.create() self.agent.rcon = lambda *_: {"ok": True, "players": 0} diff --git a/docker-compose.yml b/docker-compose.yml index 88f5ad2..2939f06 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,6 @@ services: ORIGIN: ${EC_PUBLIC_URL:-http://localhost:3000} EC_SETUP_TOKEN: ${EC_SETUP_TOKEN:-} EC_TELEMETRY_TOKEN: ${EC_TELEMETRY_TOKEN:-} - EC_BANK_TOKENS: ${EC_BANK_TOKENS:-} EC_BANK_MAX_BALANCE: ${EC_BANK_MAX_BALANCE:-250000} EC_AUTH_ALLOW_HTTP: ${EC_AUTH_ALLOW_HTTP:-false} # Legacy host imports stream whole game files; buffering routes enforce their own caps. diff --git a/docs/guides/bank.md b/docs/guides/bank.md index 5ff8aee..38dc6cc 100644 --- a/docs/guides/bank.md +++ b/docs/guides/bank.md @@ -9,32 +9,45 @@ IDs. This release does not introduce separate bank-assisted rankings. ## Setup and rollout -1. Back up the controller database. Deploy Electric Cherry first and restart - the controller. Migration 20 creates accounts, transfer requests and a ledger. -2. Generate a **different** secret for each registered game server, for example - `ec_bank_` followed by `openssl rand -hex 32`. Configure `EC_BANK_TOKENS` as a - JSON object mapping exact server names to these secrets. Docker Compose - forwards this environment variable. An example shape with placeholders is - `{"t5_zombie_theater_1":"ec_bank_"}`. +1. Back up the controller database. Update every host agent to **1.9.0 or later** + before enabling the bank. This version settles game journals before removing + runtime storage; older agents do not. Package and offline-sign the agent + release using the [host update workflow](native-hosts.md). +2. Deploy Electric Cherry and restart the controller. Migration 20 creates + accounts, requests and the ledger; migration 21 adds instance credentials and + binds existing requests to their currently active native instance, if any. 3. Set `EC_BANK_MAX_BALANCE` (default 250000, range 1–1000000). Invalid configuration - refuses new deposits. A limit reduction does not confiscate existing balances - or invalidate already prepared deposits; withdrawals remain available. + refuses new deposits. Lowering the limit preserves existing balances and + prepared deposits; withdrawals remain available. 4. Build/install TWS plugins `t5-plugin`/`t6-plugin` 1.1.0 or later and restart the **whole game-server process**. Install `tws_t5_bank.gsc` or - `tws_t6_bank.gsc` 1.0.0, the game's common module (1.0.0+) and telemetry - reporter (1.0.0+). Restart the map after installing scripts/configuration. -5. In each server's private configuration, set `tws_server_id` to its registered - name and `tws_bank_token` to that server's secret. The optional `tws_bank_url` - defaults to `https://ec.thewrightserver.net/api/bank`; overrides must use - HTTPS. Keep these settings out of shared scripts and public config downloads. + `tws_t6_bank.gsc` 1.0.0, common 1.0.0+ and telemetry 1.0.0+. +5. Create or rebuild the servers through EC. Launch preparation generates a + cryptographically random `ec_bank_` credential for each immutable instance, + stores it in the private controller database, and injects it through + `configs/electric_cherry_bank.cfg`. The launch command executes this private + file and sets `tws_server_id`. The host agent writes the private file and derives `tws_bank_url` from its + enrolled controller URL. Use an HTTPS controller URL. -Bank tokens must match `ec_bank_` followed by 32–128 URL-safe letters, digits, -underscores or hyphens. Missing, malformed, duplicate, or telemetry-reused tokens -disable authentication. `EC_TELEMETRY_TOKEN`, host tokens and browser sessions -cannot authorize bank requests. The credential determines the server, and a -different `server` in the body is refused. Rotate a token by updating both the -controller mapping and that server's private configuration, then restart them. -Keep the same server name and bank URL so existing decisions can finish. +There is **no `EC_BANK_TOKENS` environment map** and no per-server secret setup. +The private configuration is sent only in that host's authenticated operation +payload. It is excluded from shared recipes, promoted files and blob downloads. +Do not upload `electric_cherry_bank.cfg`: its filename is reserved. Protect the +controller database, host operation journal and per-instance runtime directory, +which contain private credentials. + +Retries, rebuilds and host moves of the same instance reuse its credential. +Retiring the instance immediately revokes authentication. Creating another +server with the same display name produces a new instance ID and secret; it +cannot finish the old instance's requests. Accounts remain attached to player +GUIDs, so balances survive the servers. Telemetry tokens and browser sessions +cannot authorize bank requests. Host credentials can only settle decisions +through the separate, operation-scoped teardown endpoint. + +For an upgrade from the initial static-token implementation, drain outstanding +transfers while the old controller/configuration still runs, then deploy and +rebuild. Remove the obsolete environment map and any manually supplied token. +Do not replace identities or discard pending journals to force an upgrade. ## Machine API @@ -76,7 +89,8 @@ Preparation requires the GUID to be present in the server's latest active match, with the exact telemetry match key. Late joiners may need to wait for the next 30-second heartbeat. Unknown/destroyed servers, absent players and old matches cannot prepare transfers. Settlement of an existing decision remains possible -after match completion, retention or server deletion. +after match completion or retention. Host teardown can finish saved decisions +after retirement; the retired game credential itself no longer authenticates. Successful application responses include `ok`, `state`, `code`, `request_id`, `amount` and `balance`. States are `balance`, `prepared`, `committed`, `cancelled` @@ -110,7 +124,8 @@ decision, retries only finish the ledger; they never change game points again. These points now belong to the current match, so an applied withdrawal is not automatically refunded if that match ends or its server crashes. -Keep `fs_homepath/tws-bank/t5/` or `t6/` on persistent writable storage with a +Keep `tws-bank/t5/` or `tws-bank/t6/`, relative to the game process working +directory, on persistent writable storage with a single process per server/endpoint. Journals contain transfer data, not secrets. On a map/process restart, saved decisions settle before new bank commands. Unreadable storage disables the bank; failed writes prevent points changes. @@ -121,6 +136,18 @@ original journal to resolve holds; never delete a pending journal to unblock the bank. If storage is permanently lost, reconcile against the retained request and ledger records before any manual repair. +During removal, rebuild, host migration or launch cancellation, agent 1.9.0+ +stops the game and drains its journals **before** deleting runtime files or +starting a replacement. It uses its host credential at +`POST /api/agent/bank/settle`; EC checks the assigned host, immutable instance, +unfinished teardown operation, server and original request identity. This +endpoint permits only commit/cancel, never new deposits or withdrawals. Journal +URLs must match that host's configured controller; the agent never follows a +journal URL. Corruption, a mismatched controller, failed acknowledgment or an +outage retains storage and leaves the operation unresolved for retry. An +interrupted rebuild uses the existing rebuild recovery workflow to retry with +a fresh operation after settlement. Do not manually erase its old directory. + The journal provides recovery from map/process restart. It cannot make live game memory and SQLite one atomic transaction or preserve match points after a machine/storage failure. Back up persistent storage and verify crash behavior diff --git a/docs/guides/native-hosts.md b/docs/guides/native-hosts.md index dd54e62..f8758b5 100644 --- a/docs/guides/native-hosts.md +++ b/docs/guides/native-hosts.md @@ -1035,3 +1035,12 @@ game hosts. Run: - Controller restart, agent restart and network loss before execution, during execution and after execution but before acknowledgement. Verify the same operation IDs and absence of duplicate containers/restarts. + +### Bank journals during server teardown + +Agent 1.9.0 adds `bank.settlement`: removal, rebuild, migration and creation +cancellation finish pending TWS bank decisions before deleting the old runtime. +An unavailable controller or corrupt journal leaves the operation unresolved +and preserves storage. Update all agents before enabling the +[player bank](bank.md). EC generates and injects bank credentials per instance; +no token map or manual server-secret provisioning is required. diff --git a/src/lib/server/auth/bank.ts b/src/lib/server/auth/bank.ts index 936298d..8469a30 100644 --- a/src/lib/server/auth/bank.ts +++ b/src/lib/server/auth/bank.ts @@ -1,41 +1,46 @@ -import { createHash, timingSafeEqual } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; +import { db } from '../db/index.js'; -const digest = (value: string) => createHash('sha256').update(value).digest(); -const valid = (value: unknown): value is string => - typeof value === 'string' && /^ec_bank_[A-Za-z0-9_-]{32,128}$/.test(value); +const digest = (value: string) => createHash('sha256').update(value).digest('hex'); -/** Dedicated, unique credentials; telemetry and browser credentials never authorize spending. */ +/** Private controller state, reused only by the same immutable native instance. */ +export function bankCredential(instanceId: string): string { + return db() + .transaction(() => { + const instance = db() + .prepare('SELECT server FROM native_instances WHERE id=? AND retired_at IS NULL') + .get(instanceId) as { server: string } | undefined; + if (!instance) throw new Error('Bank credentials require an active native instance.'); + const saved = db() + .prepare('SELECT token FROM bank_credentials WHERE instance_id=?') + .get(instanceId) as { token: string } | undefined; + if (saved) return saved.token; + const token = 'ec_bank_' + randomBytes(32).toString('base64url'); + db() + .prepare( + 'INSERT INTO bank_credentials(instance_id,server,token,token_hash,created_at) VALUES (?,?,?,?,?)' + ) + .run(instanceId, instance.server, token, digest(token), new Date().toISOString()); + return token; + }) + .immediate(); +} + +/** Retiring an instance immediately revokes its game credential, including reads. */ export function authenticateBank( request: Request -): { ok: true; server: string } | { ok: false; status: 401 | 503; code: string } { - let entries: [string, string][]; - try { - const config: unknown = JSON.parse(process.env.EC_BANK_TOKENS ?? '{}'); - if (!config || typeof config !== 'object' || Array.isArray(config)) throw new Error(); - const values = Object.entries(config); - if ( - !values.length || - values.some( - ([server, token]) => - !server || !valid(token) || token === process.env.EC_TELEMETRY_TOKEN - ) - ) - throw new Error(); - entries = values as [string, string][]; - if (new Set(entries.map(([, token]) => token)).size !== entries.length) throw new Error(); - } catch { - return { ok: false, status: 503, code: 'unconfigured' }; - } +): { ok: true; server: string; instanceId: string } | { ok: false; status: 401; code: string } { const token = request.headers .get('authorization') ?.match(/^Bearer (ec_bank_[A-Za-z0-9_-]{32,128})$/i)?.[1]; - if (!token) return { ok: false, status: 401, code: 'unauthorized' }; - const supplied = digest(token); - let matched: string | undefined; - for (const [server, expected] of entries) { - if (timingSafeEqual(supplied, digest(expected))) matched = server; - } - return matched - ? { ok: true, server: matched } - : { ok: false, status: 401, code: 'unauthorized' }; + const matched = token + ? (db() + .prepare( + `SELECT c.server, c.instance_id AS instanceId + FROM bank_credentials c JOIN native_instances n ON n.id=c.instance_id AND n.server=c.server + WHERE c.token_hash=? AND n.retired_at IS NULL` + ) + .get(digest(token)) as { server: string; instanceId: string } | undefined) + : undefined; + return matched ? { ok: true, ...matched } : { ok: false, status: 401, code: 'unauthorized' }; } diff --git a/src/lib/server/db/bank-credentials.ts b/src/lib/server/db/bank-credentials.ts new file mode 100644 index 0000000..3aad994 --- /dev/null +++ b/src/lib/server/db/bank-credentials.ts @@ -0,0 +1,22 @@ +import type Database from 'better-sqlite3'; + +export function migrateBankCredentials(handle: Database.Database): void { + handle.exec(` + CREATE TABLE IF NOT EXISTS bank_credentials ( + instance_id TEXT PRIMARY KEY, server TEXT NOT NULL, + token TEXT NOT NULL, token_hash TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL + ); + `); + if ( + !(handle.pragma('table_info(bank_requests)') as { name: string }[]).some( + (column) => column.name === 'instance_id' + ) + ) { + handle.exec(` + ALTER TABLE bank_requests ADD COLUMN instance_id TEXT; + UPDATE bank_requests SET instance_id = ( + SELECT id FROM native_instances WHERE server = bank_requests.server AND retired_at IS NULL + ); + `); + } +} diff --git a/src/lib/server/db/migrations.ts b/src/lib/server/db/migrations.ts index 8fda343..df6fd6f 100644 --- a/src/lib/server/db/migrations.ts +++ b/src/lib/server/db/migrations.ts @@ -19,6 +19,7 @@ import { migrateGameFileSchema } from '../game-files/schema.js'; import { migrateLeaderboardReports } from './leaderboard-reports.js'; import { migratePlayerScoreTotal } from './player-score-total.js'; import { migrateBank } from './bank.js'; +import { migrateBankCredentials } from './bank-credentials.js'; import type { Migration } from './migration-runner.js'; // Append only: never renumber entries or change an applied migration to add an @@ -111,7 +112,8 @@ export const migrations: readonly Migration[] = [ name: 'cumulative player score per match', run: migratePlayerScoreTotal }, - { version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank } + { version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank }, + { version: 21, name: 'instance-owned bank credentials', run: migrateBankCredentials } ]; export const databaseVersion = migrations.length; diff --git a/src/lib/server/fleet/bank.ts b/src/lib/server/fleet/bank.ts index 7ad5eef..5a255b0 100644 --- a/src/lib/server/fleet/bank.ts +++ b/src/lib/server/fleet/bank.ts @@ -20,6 +20,7 @@ interface Result { balance: number; } interface Saved { + instance_id: string | null; fingerprint: string; state: string; amount: number; @@ -79,7 +80,8 @@ export function bankProfile(guid: string) { export function bankRequest( action: BankAction, value: unknown, - authenticatedServer: string + authenticatedServer: string, + instanceId: string ): { status: number; body: Result } { const result = (status: number, code: string, request_id = '') => ({ status, @@ -121,7 +123,7 @@ export function bankRequest( const saved = handle .prepare('SELECT * FROM bank_requests WHERE server = ? AND request_id = ?') .get(server, request_id) as Saved | undefined; - if (saved && saved.fingerprint !== fingerprint) + if (saved && (saved.instance_id !== instanceId || saved.fingerprint !== fingerprint)) return result(409, 'request_conflict', request_id); const balance = ( @@ -145,8 +147,8 @@ export function bankRequest( const insert = (body: Result, matchId: number | null = null) => { handle .prepare( - `INSERT INTO bank_requests(server,request_id,guid,fingerprint,operation,match_id,amount,state,result,created_at) - VALUES (?,?,?,?,?,?,?,?,?,?)` + `INSERT INTO bank_requests(server,request_id,guid,fingerprint,operation,match_id,amount,state,result,created_at,instance_id) + VALUES (?,?,?,?,?,?,?,?,?,?,?)` ) .run( server, @@ -158,7 +160,8 @@ export function bankRequest( body.amount, body.state, JSON.stringify(body), - now + now, + instanceId ); return { status: 200, body }; }; @@ -200,6 +203,14 @@ export function bankRequest( return { status: 200, body }; } if (saved) return { status: 200, body: JSON.parse(saved.result) as Result }; + if ( + !handle + .prepare( + 'SELECT 1 FROM native_instances WHERE id=? AND server=? AND retired_at IS NULL' + ) + .get(instanceId, server) + ) + return result(403, 'retired_instance', request_id); const member = handle .prepare( `SELECT m.id FROM leaderboard_matches m diff --git a/src/lib/server/provisioning/launch-preparation.ts b/src/lib/server/provisioning/launch-preparation.ts index a9f9f6a..da95cb4 100644 --- a/src/lib/server/provisioning/launch-preparation.ts +++ b/src/lib/server/provisioning/launch-preparation.ts @@ -13,6 +13,7 @@ export interface NativeLaunchInput { instance: NativeInstance; serverKey: string; rconPassword: string; + bankToken: string; rebuild: boolean; safetyOverride?: boolean; unhealthyRecovery?: boolean; diff --git a/src/lib/server/provisioning/launch-worker.ts b/src/lib/server/provisioning/launch-worker.ts index ede2052..45bc3da 100644 --- a/src/lib/server/provisioning/launch-worker.ts +++ b/src/lib/server/provisioning/launch-worker.ts @@ -71,6 +71,19 @@ parentPort!.on( }; })(); const instance = input.instance; + // Credential files travel only in this host's private operation payload, never + // shared recipes, promoted files, or the content-addressed download store. + const bankPath = posix.join( + relDestForKind('config', recipe.game), + 'electric_cherry_bank.cfg' + ); + if ( + recipe.files.some( + (file) => + posix.join(relDestForKind(file.kind, recipe.game), file.path) === bankPath + ) + ) + throw new Error('electric_cherry_bank.cfg is reserved for instance credentials.'); const payload = { name: instance.server, game: recipe.game, @@ -79,13 +92,14 @@ parentPort!.on( port: instance.port, bindAddress: instance.bind_address, rconPassword: password, + bank: { token: input.bankToken, configPath: bankPath }, env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port), PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp', PLUTAINER_CONFIG_FILE: recipe.configFile, PLUTAINER_MAP_ROTATE: 'false', - PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, + PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +exec electric_cherry_bank.cfg +set tws_server_id ${instance.server} +map_rotate`, PLUTO_SERVER_KEY: input.serverKey }, files: recipe.files.map((file) => ({ diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 968cadf..c469b6a 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -12,6 +12,7 @@ import { isIP } from 'node:net'; import { setImmediate as yieldToController } from 'node:timers/promises'; import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js'; import { randomBytes, randomUUID } from 'node:crypto'; +import { bankCredential } from '../auth/bank.js'; import { posix } from 'node:path'; import { db } from '../db/index.js'; import { unsettledCancelledCommand } from './cancellation.js'; @@ -1936,6 +1937,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { instance, serverKey: preparedKey, rconPassword: preparedPassword, + bankToken: bankCredential(instance.id), rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety, unhealthyRecovery: host.inventory.capabilities?.includes( diff --git a/src/routes/api/agent/bank/settle/+server.ts b/src/routes/api/agent/bank/settle/+server.ts new file mode 100644 index 0000000..109fb3f --- /dev/null +++ b/src/routes/api/agent/bank/settle/+server.ts @@ -0,0 +1,28 @@ +import { error, json } from '@sveltejs/kit'; +import { authenticateAgent, objectBody } from '$lib/server/hosts/http.js'; +import { db } from '$lib/server/db/index.js'; +import { bankRequest } from '$lib/server/fleet/bank.js'; + +/** An assigned teardown may finish durable decisions, never prepare new spending. */ +export async function POST({ request }: { request: Request }) { + const host = authenticateAgent(request); + const body = await objectBody(request, 8192); + if (typeof body.operationId !== 'string' || !['commit', 'cancel'].includes(String(body.action))) + error(400, 'Invalid bank settlement.'); + const scope = db() + .prepare( + `SELECT n.id, n.server FROM ec_host_operations o + JOIN native_instances n ON n.id=o.instance_id + WHERE o.id=? AND o.host_id=? AND o.kind IN ('instance.remove','instance.rebuild','instance.cancel-create') + AND o.state IN ('pending','accepted','running','unknown')` + ) + .get(body.operationId, host.id) as { id: string; server: string } | undefined; + if (!scope) error(403, 'Bank settlement requires an assigned, unfinished teardown.'); + const result = bankRequest( + body.action as 'commit' | 'cancel', + body.transfer, + scope.server, + scope.id + ); + return json(result.body, { status: result.status, headers: { 'Cache-Control': 'no-store' } }); +} diff --git a/src/routes/api/bank/[action]/+server.ts b/src/routes/api/bank/[action]/+server.ts index 4cb1d95..97260ab 100644 --- a/src/routes/api/bank/[action]/+server.ts +++ b/src/routes/api/bank/[action]/+server.ts @@ -29,7 +29,8 @@ export async function POST({ request, params }: { request: Request; params: { ac const { body, status } = bankRequest( params.action as BankAction, value, - authentication.server + authentication.server, + authentication.instanceId ); return json(body, { status, headers }); } catch { diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py index 90d8bb0..be5dc3f 100644 --- a/static/agent/ec-agent.py +++ b/static/agent/ec-agent.py @@ -35,7 +35,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.8.7" +VERSION = "1.9.0" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -1503,7 +1503,7 @@ class Agent: if line.startswith("PRETTY_NAME="): os_name = line.split("=", 1)[1].strip('"') update = self.update_inventory() - capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) + capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) plutonium_cache = self.plutonium_cache_inventory() return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} @@ -1657,7 +1657,7 @@ class Agent: # record. Whenever preparation was recorded, enforce that exact image. if image in prepared and prepared[image] != image_id: raise AgentError("Prepared runtime image has changed. An admin must prepare it again before launching.") - files = payload.get("files", []) + files = self.initial_files(payload) if not isinstance(files, list) or len(files) > 10000: raise Rejected("Invalid initial file list") total, paths = 0, set() @@ -1911,7 +1911,7 @@ class Agent: self.operation_phase = "seeding-runtime" self.seed_plutonium(app, image) self.operation_phase = "writing-create-files" - self.write_initial_files(app, payload.get("files", [])) + self.write_initial_files(app, self.initial_files(payload)) self.operation_phase = "persisting-create-metadata" atomic_json(directory / "spec.json", payload) env_path = directory / "environment" @@ -1944,6 +1944,25 @@ class Agent: if self.executing_id and self.executing_id in self.cancelled_creations: raise AgentError("Creation cancelled; waiting for instance cleanup") + def initial_files(self, payload): + files = payload.get("files", []) + if not isinstance(files, list): + raise Rejected("Invalid initial file list") + bank = payload.get("bank") + if bank is None: + return files + if not isinstance(bank, dict) or not isinstance(bank.get("token"), str) or not re.fullmatch(r"ec_bank_[A-Za-z0-9_-]{32,128}", bank["token"]): + raise Rejected("Invalid instance bank credential") + path = bank.get("configPath") + scoped(self.root / "agent" / "validation", path) + if PurePosixPath(path).name != "electric_cherry_bank.cfg": + raise Rejected("Invalid private bank config path") + controller = self.config["controller"].rstrip("/") + if any(char in controller for char in ('"', "\\", "\n", "\r", "\x00")): + raise Rejected("Invalid bank controller URL") + content = f'set tws_bank_token "{bank["token"]}"\nset tws_bank_url "{controller}/api/bank"\n'.encode() + return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}] + def write_initial_files(self, root, files): if not isinstance(files, list) or len(files) > 10000: raise AgentError("Invalid file list") @@ -1992,6 +2011,72 @@ class Agent: if (time.monotonic() - started) * 1000 >= ttl: raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation") + def settle_bank(self, instance_id, generation, operation_id=None): + """Drain stopped runtime journals before discarding any generation's storage. + + Use only our configured controller and host credential. Never follow an + endpoint or symlink supplied by a game journal. Lost replies are retried + with the original request identity; files remain until teardown succeeds. + """ + app = scoped(self.instance_dir(instance_id, generation), "app") + if not app.exists(): + return + if self.inspect(instance_id, generation)["running"]: + raise AgentError("Bank settlement requires a stopped instance") + operation_id = operation_id or self.executing_id + visited = 0 + deadline = time.monotonic() + 30 + journals = [] + + def unreadable(_error): + raise AgentError("Bank journal discovery failed; runtime storage retained") + + for directory, directories, files in os.walk(app, followlinks=False, onerror=unreadable): + relative = Path(directory).relative_to(app) + inside_bank = "tws-bank" in relative.parts + visited += 1 + if visited > 100000 or time.monotonic() > deadline: + raise AgentError("Bank journal discovery limit exceeded; runtime storage retained") + for name in directories + files: + if (inside_bank or name == "tws-bank") and (Path(directory) / name).is_symlink(): + raise AgentError("Bank journal paths must not be symlinks") + if not inside_bank: + continue + for name in files: + if name.endswith(".json"): + journals.append((relative / name).as_posix()) + if len(journals) > 4096: + raise AgentError("Too many bank journals; runtime storage retained") + for path in sorted(journals): + try: + raw = managed_read(app, path) + except Rejected: + raise AgentError("Bank journal cannot be read; runtime storage retained") from None + if raw is None or len(raw) > 16 * 1024 * 1024: + raise AgentError("Bank journal is unavailable or oversized") + try: + journal = json.loads(raw) + reports = journal["reports"] + if journal["endpoint"].rstrip("/") != self.config["controller"].rstrip("/") + "/api/bank": + raise AgentError("Bank journal belongs to a different controller; runtime storage retained") + if type(journal["version"]) is not int or journal["version"] != 1 or not isinstance(reports, list) or len(reports) > 4096: + raise ValueError() + # Controller independently binds this operation to the instance/server. + for report in reports: + if not isinstance(report, str) or len(report.encode()) > 20000: + raise ValueError() + transfer = json.loads(report) + action = transfer["decision"] + if action not in ("commit", "cancel") or transfer["server"] != journal["server"] or not operation_id: + raise ValueError() + response = self.request("POST", "/api/agent/bank/settle", { + "operationId": operation_id, "action": action, "transfer": transfer}) + expected = ("committed",) if action == "commit" else ("cancelled", "rejected") + if not isinstance(response, dict) or response.get("request_id") != transfer["request_id"] or response.get("state") not in expected: + raise AgentError("Bank settlement was not acknowledged; runtime storage retained") + except (ValueError, TypeError, KeyError, AttributeError): + raise AgentError("Bank journal is invalid; runtime storage retained") from None + def lifecycle(self, kind, instance_id, generation, payload): unhealthy_recovery = payload.get("unhealthyRecovery") is True if unhealthy_recovery and (kind not in ("instance.restart", "instance.rebuild") or payload.get("stageId") or "autoscaling" in payload): @@ -2043,6 +2128,7 @@ class Agent: if health["exists"]: self.docker.call("container", "stop", "--time", "30", name) self.docker.call("container", "rm", name) + self.settle_bank(instance_id, generation) if kind == "instance.rebuild": result = self.create(instance_id, next_generation, spec) self.operation_phase = "removing-old-instance" @@ -2062,7 +2148,7 @@ class Agent: raise AgentError("Unsupported instance operation") return {**self.inspect(instance_id, generation), "before": before} - def cancel_creation(self, instance_id, generation, payload): + def cancel_creation(self, instance_id, generation, payload, operation_id=None): """Explicit launch cancellation, fenced to its immutable creation identity. The operation loop serializes this with create. Once delivered, removal @@ -2106,6 +2192,7 @@ class Agent: self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, target_generation)) if self.inspect(instance_id, target_generation)["exists"]: raise AgentError("Cancelled creation container still exists") + self.settle_bank(instance_id, target_generation, operation_id) directory = self.instance_dir(instance_id, target_generation) if directory.exists(): shutil.rmtree(directory) @@ -2747,6 +2834,7 @@ class Agent: if not unchanged_generation() or not both_missing(): return row + self.settle_bank(instance_id, generation, operation["id"]) # Preparation can leave runtime/configuration files before create's # checkpoint. A fresh operation must not inherit those partial files. expected_spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")} @@ -2793,7 +2881,7 @@ class Agent: if kind == "instance.remove" and "autoscaling" in payload and not checkpoint: return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision") if kind == "instance.cancel-create": - return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload)) + return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload, operation["id"])) if kind == "instance.create": marker = self.instance_dir(instance_id, generation).parent / "current.json" if marker.exists(): @@ -2885,6 +2973,7 @@ class Agent: directory = self.instance_dir(instance_id, observed_generation) atomic_json(directory.parent / "current.json", {"generation": observed_generation}) if kind == "instance.rebuild" and not self.inspect(instance_id, generation)["exists"]: + self.settle_bank(instance_id, generation, operation["id"]) old = self.instance_dir(instance_id, generation) if old.exists(): shutil.rmtree(old) @@ -2899,6 +2988,7 @@ class Agent: elif kind == "instance.remove": proven = not health["exists"] if proven: + self.settle_bank(instance_id, generation, operation["id"]) directory = self.instance_dir(instance_id, generation) if directory.exists(): shutil.rmtree(directory) diff --git a/static/agent/manifest.json b/static/agent/manifest.json index 515a6f5..4688fcf 100644 --- a/static/agent/manifest.json +++ b/static/agent/manifest.json @@ -1,8 +1,8 @@ { - "version": "1.8.7", + "version": "1.9.0", "protocolVersion": 1, "platform": "debian-amd64", - "sha256": "e773c907141c97eab2055b354306973b248a67109a51337f1f1d76d80aaa8701", + "sha256": "07f91ba8780c1e2b0a9c0ab3534c1731d4246d3f5d84fece14691c930dcb5e90", "updaterProtocolVersion": 2, "updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb", "installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231" diff --git a/static/agent/manifest.json.sig b/static/agent/manifest.json.sig deleted file mode 100644 index dead811..0000000 --- a/static/agent/manifest.json.sig +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN PGP SIGNATURE----- - -iJEEABYKADkWIQSpocb6VBfQzvdIh9HF6fdSNniN+gUCasmr1RsUgAAAAAAEAA5t -YW51MiwyLjUrMS4xMiwwLDMACgkQxen3UjZ4jfprZQD+PPqrtEYWTImhdy1eFAfv -k5XD1IaFOgSvcsL9P0tj69IBAIRGt3zqWw9E8vFeyZXz58GfnscyFI6ws5ln+l3h -i2UP -=Er1r ------END PGP SIGNATURE----- diff --git a/tests/unit/bank.test.ts b/tests/unit/bank.test.ts index a1fb18b..dc28b42 100644 --- a/tests/unit/bank.test.ts +++ b/tests/unit/bank.test.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'node:crypto'; import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { routePermission } from '../../src/lib/server/auth/access.js'; @@ -5,20 +6,38 @@ import { routePermission } from '../../src/lib/server/auth/access.js'; let fixture: Fixture; const server = 't5_zombie_theater_1'; const other = 't6_zm_transit_1'; -const secret = 'ec_bank_' + 'a'.repeat(40); -const otherSecret = 'ec_bank_' + 'b'.repeat(40); +let secret: string; +let otherSecret: string; +let instanceId: string; +let hostId: string; +let hostToken: string; let serial = 0; beforeEach(async () => { vi.resetModules(); fixture = await newFixture([server, other]); - vi.stubEnv('EC_BANK_TOKENS', JSON.stringify({ [server]: secret, [other]: otherSecret })); vi.stubEnv('EC_TELEMETRY_TOKEN', 'telemetry_' + 't'.repeat(40)); vi.stubEnv('EC_BANK_MAX_BALANCE', '250000'); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js'); await seedServerRegistry(); + const { db } = await import('../../src/lib/server/db/index.js'); + const { bankCredential } = await import('../../src/lib/server/auth/bank.js'); + const { createHost, createEnrollment, enrollHost } = + await import('../../src/lib/server/hosts/index.js'); + hostId = createHost({ name: 'Bank host' }).id; + hostToken = enrollHost(createEnrollment(hostId).token, {}).credential; for (const source of [server, other]) { + const id = randomUUID(); + db() + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES (?,?,?,'library','image',28960,?)` + ) + .run(id, source, hostId, new Date().toISOString()); + if (source === server) { + instanceId = id; + secret = bankCredential(id); + } else otherSecret = bankCredential(id); saveTelemetry({ server: source, map: source === server ? 'zombie_theater' : 'zm_transit', @@ -84,21 +103,88 @@ it('routes only bank POSTs as machine requests, authenticating before reading bo }); }); -it('fails closed for missing, duplicate, telemetry-reused and invalid credentials', async () => { - for (const configured of [ - '', - '{}', - '[]', - '{', - JSON.stringify({ [server]: secret, [other]: secret }), - JSON.stringify({ [server]: 'short' }) - ]) { - vi.stubEnv('EC_BANK_TOKENS', configured); - expect(await post('balance', { server, guid: 'ace' })).toMatchObject({ status: 503 }); - } - vi.stubEnv('EC_BANK_TOKENS', JSON.stringify({ [server]: secret })); - vi.stubEnv('EC_TELEMETRY_TOKEN', secret); - expect(await post('balance', { server, guid: 'ace' })).toMatchObject({ status: 503 }); +it('owns generated secrets by immutable instance, preserving rebuilds and revoking retired names', async () => { + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const { bankCredential } = await import('../../src/lib/server/auth/bank.js'); + const data = transfer(); + await post('deposit', data); + expect(secret).toMatch(/^ec_bank_[A-Za-z0-9_-]{43}$/); + expect(otherSecret).not.toBe(secret); + closeDb(); + db().prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId); + expect(bankCredential(instanceId)).toBe(secret); + db() + .prepare('UPDATE native_instances SET retired_at=? WHERE id=?') + .run(new Date().toISOString(), instanceId); + expect(await post('balance', { server, guid: 'ace' })).toMatchObject({ status: 401 }); + expect(() => bankCredential(instanceId)).toThrow('active native instance'); + const next = randomUUID(); + db() + .prepare( + `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES (?,?,?,'library','image',28960,?)` + ) + .run(next, server, hostId, new Date().toISOString()); + const replacement = bankCredential(next); + const { migrateBankCredentials } = await import('../../src/lib/server/db/bank-credentials.js'); + migrateBankCredentials(db()); // A legacy migration rerun must not rebind old requests. + expect(replacement).not.toBe(secret); + expect(await post('commit', data, replacement)).toMatchObject({ + status: 409, + code: 'request_conflict' + }); + expect(await post('cancel', data, replacement)).toMatchObject({ status: 409 }); +}); + +it('settles journals only for the host assigned to an unfinished teardown, even after retirement', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { queueHostOperation } = await import('../../src/lib/server/hosts/index.js'); + const { POST } = await import('../../src/routes/api/agent/bank/settle/+server.js'); + const data = transfer(); + await post('deposit', data); + const removal = queueHostOperation( + hostId, + 'instance.remove', + {}, + { instanceId, generation: 1 } + ); + db() + .prepare('UPDATE native_instances SET retired_at=? WHERE id=?') + .run(new Date().toISOString(), instanceId); + const settle = ( + action = 'commit', + operationId = removal.id, + credential = hostToken, + value = data + ) => + POST({ + request: new Request('https://test/api/agent/bank/settle', { + method: 'POST', + headers: { Authorization: `Bearer ${credential}` }, + body: JSON.stringify({ operationId, action, transfer: value }) + }) + }); + await expect(settle('commit', removal.id, secret)).rejects.toMatchObject({ status: 401 }); + const { createHost, createEnrollment, enrollHost } = + await import('../../src/lib/server/hosts/index.js'); + const stranger = createHost({ name: 'Unassigned host' }); + const strangerToken = enrollHost(createEnrollment(stranger.id).token, {}).credential; + await expect(settle('commit', removal.id, strangerToken)).rejects.toMatchObject({ + status: 403 + }); + await expect(settle('deposit')).rejects.toMatchObject({ status: 400 }); + await expect(settle('commit', randomUUID())).rejects.toMatchObject({ status: 403 }); + expect( + await (await settle('commit', removal.id, hostToken, { ...data, server: other })).json() + ).toMatchObject({ code: 'wrong_server' }); + const first = await (await settle()).json(); + expect(first).toMatchObject({ state: 'committed', balance: 2000 }); + expect(await (await settle()).json()).toEqual(first); + const cancelled = transfer(); + expect(await (await settle('cancel', removal.id, hostToken, cancelled)).json()).toMatchObject({ + state: 'cancelled' + }); + db().prepare("UPDATE ec_host_operations SET state='completed' WHERE id=?").run(removal.id); + await expect(settle()).rejects.toMatchObject({ status: 403 }); }); it('persists one credit and one ledger row through lost replies, retries and database reopen', async () => { diff --git a/tests/unit/native-launch-preparation.test.ts b/tests/unit/native-launch-preparation.test.ts index 60e629b..76beff2 100644 --- a/tests/unit/native-launch-preparation.test.ts +++ b/tests/unit/native-launch-preparation.test.ts @@ -416,3 +416,47 @@ describe('asynchronous native launch preparation', () => { expect(await operationFor(second.deploymentId)).toBeDefined(); }); }); + +it('injects unique persistent bank credentials only into private runtime files', async () => { + const job = await queue(); + await runPass(); + const op = await operationFor(job.deploymentId); + const { getHostOperation } = await import('../../src/lib/server/hosts/index.js'); + const { nativeInstance } = await import('../../src/lib/server/backends/native.js'); + const { bankCredential, authenticateBank } = await import('../../src/lib/server/auth/bank.js'); + const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const instance = nativeInstance(job.name)!; + const credential = bankCredential(instance.id); + const payload = getHostOperation(op!.id)!.payload as { + bank: { token: string; configPath: string }; + files: { path: string }[]; + env: Record; + }; + expect(payload.bank).toEqual({ + token: credential, + configPath: 'configs/electric_cherry_bank.cfg' + }); + expect(payload.files.some((file) => file.path.endsWith('electric_cherry_bank.cfg'))).toBe( + false + ); + const { db } = await import('../../src/lib/server/db/index.js'); + expect( + db() + .prepare('SELECT count(*) AS n FROM file_blobs WHERE instr(content,?) > 0') + .get(Buffer.from(credential)) + ).toEqual({ n: 0 }); + expect(payload.env.PLUTAINER_EXTRA_ARGS).toContain( + '+exec electric_cherry_bank.cfg +set tws_server_id' + ); + expect( + authenticateBank( + new Request('https://test', { headers: { Authorization: `Bearer ${credential}` } }) + ) + ).toMatchObject({ ok: true, server: job.name, instanceId: instance.id }); + expect(JSON.stringify(recipeForDeployment(job.deploymentId))).not.toContain(credential); + expect( + recipeForDeployment(job.deploymentId)!.recipe.files.some( + (file) => file.path === 'electric_cherry_bank.cfg' + ) + ).toBe(false); +}); diff --git a/tests/unit/runtime-config.test.ts b/tests/unit/runtime-config.test.ts index 26f6a04..7c5acca 100644 --- a/tests/unit/runtime-config.test.ts +++ b/tests/unit/runtime-config.test.ts @@ -149,6 +149,7 @@ it.each([ instance: backend.nativeInstance(a.server)!, serverKey: 'test-key', rconPassword: password, + bankToken: 'ec_bank_' + 'a'.repeat(43), rebuild: false }); const actualConfig = prepared.recipe.files.find((file) => file.path === target)!;