Provision instance bank credentials automatically (Refs #146)

.env.example

@@ -7,9 +7,7 @@ EC_SETUP_TOKEN=
 # Game reporters send Authorization: Bearer <token> or X-EC-Telemetry-Token.
 # Telemetry ingestion is disabled until a token of at least 32 chars is set.
 EC_TELEMETRY_TOKEN=
-# Unique per-server bank secrets: JSON object mapping exact server names to
+# Bank credentials are generated and injected automatically per native instance.
-# ec_bank_<32+ random URL-safe characters>. Never reuse the telemetry secret.
-EC_BANK_TOKENS={}
 EC_BANK_MAX_BALANCE=250000
 # Persistent SQLite metadata, game-file storage, and beta archives.
 # For an existing deployment, retain its original data directory.

agent/ec_agent.py

@@ -35,7 +35,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.8.7"
+VERSION = "1.9.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1503,7 +1503,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         plutonium_cache = self.plutonium_cache_inventory()
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
@@ -1657,7 +1657,7 @@ class Agent:
         # record. Whenever preparation was recorded, enforce that exact image.
         if image in prepared and prepared[image] != image_id:
             raise AgentError("Prepared runtime image has changed. An admin must prepare it again before launching.")
-        files = payload.get("files", [])
+        files = self.initial_files(payload)
         if not isinstance(files, list) or len(files) > 10000:
             raise Rejected("Invalid initial file list")
         total, paths = 0, set()
@@ -1911,7 +1911,7 @@ class Agent:
         self.operation_phase = "seeding-runtime"
         self.seed_plutonium(app, image)
         self.operation_phase = "writing-create-files"
-        self.write_initial_files(app, payload.get("files", []))
+        self.write_initial_files(app, self.initial_files(payload))
         self.operation_phase = "persisting-create-metadata"
         atomic_json(directory / "spec.json", payload)
         env_path = directory / "environment"
@@ -1944,6 +1944,25 @@ class Agent:
         if self.executing_id and self.executing_id in self.cancelled_creations:
             raise AgentError("Creation cancelled; waiting for instance cleanup")
 
+    def initial_files(self, payload):
+        files = payload.get("files", [])
+        if not isinstance(files, list):
+            raise Rejected("Invalid initial file list")
+        bank = payload.get("bank")
+        if bank is None:
+            return files
+        if not isinstance(bank, dict) or not isinstance(bank.get("token"), str) or not re.fullmatch(r"ec_bank_[A-Za-z0-9_-]{32,128}", bank["token"]):
+            raise Rejected("Invalid instance bank credential")
+        path = bank.get("configPath")
+        scoped(self.root / "agent" / "validation", path)
+        if PurePosixPath(path).name != "electric_cherry_bank.cfg":
+            raise Rejected("Invalid private bank config path")
+        controller = self.config["controller"].rstrip("/")
+        if any(char in controller for char in ('"', "\\", "\n", "\r", "\x00")):
+            raise Rejected("Invalid bank controller URL")
+        content = f'set tws_bank_token "{bank["token"]}"\nset tws_bank_url "{controller}/api/bank"\n'.encode()
+        return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}]
+
     def write_initial_files(self, root, files):
         if not isinstance(files, list) or len(files) > 10000:
             raise AgentError("Invalid file list")
@@ -1992,6 +2011,72 @@ class Agent:
         if (time.monotonic() - started) * 1000 >= ttl:
             raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation")
 
+    def settle_bank(self, instance_id, generation, operation_id=None):
+        """Drain stopped runtime journals before discarding any generation's storage.
+
+        Use only our configured controller and host credential. Never follow an
+        endpoint or symlink supplied by a game journal. Lost replies are retried
+        with the original request identity; files remain until teardown succeeds.
+        """
+        app = scoped(self.instance_dir(instance_id, generation), "app")
+        if not app.exists():
+            return
+        if self.inspect(instance_id, generation)["running"]:
+            raise AgentError("Bank settlement requires a stopped instance")
+        operation_id = operation_id or self.executing_id
+        visited = 0
+        deadline = time.monotonic() + 30
+        journals = []
+
+        def unreadable(_error):
+            raise AgentError("Bank journal discovery failed; runtime storage retained")
+
+        for directory, directories, files in os.walk(app, followlinks=False, onerror=unreadable):
+            relative = Path(directory).relative_to(app)
+            inside_bank = "tws-bank" in relative.parts
+            visited += 1
+            if visited > 100000 or time.monotonic() > deadline:
+                raise AgentError("Bank journal discovery limit exceeded; runtime storage retained")
+            for name in directories + files:
+                if (inside_bank or name == "tws-bank") and (Path(directory) / name).is_symlink():
+                    raise AgentError("Bank journal paths must not be symlinks")
+            if not inside_bank:
+                continue
+            for name in files:
+                if name.endswith(".json"):
+                    journals.append((relative / name).as_posix())
+                    if len(journals) > 4096:
+                        raise AgentError("Too many bank journals; runtime storage retained")
+        for path in sorted(journals):
+            try:
+                raw = managed_read(app, path)
+            except Rejected:
+                raise AgentError("Bank journal cannot be read; runtime storage retained") from None
+            if raw is None or len(raw) > 16 * 1024 * 1024:
+                raise AgentError("Bank journal is unavailable or oversized")
+            try:
+                journal = json.loads(raw)
+                reports = journal["reports"]
+                if journal["endpoint"].rstrip("/") != self.config["controller"].rstrip("/") + "/api/bank":
+                    raise AgentError("Bank journal belongs to a different controller; runtime storage retained")
+                if type(journal["version"]) is not int or journal["version"] != 1 or not isinstance(reports, list) or len(reports) > 4096:
+                    raise ValueError()
+                # Controller independently binds this operation to the instance/server.
+                for report in reports:
+                    if not isinstance(report, str) or len(report.encode()) > 20000:
+                        raise ValueError()
+                    transfer = json.loads(report)
+                    action = transfer["decision"]
+                    if action not in ("commit", "cancel") or transfer["server"] != journal["server"] or not operation_id:
+                        raise ValueError()
+                    response = self.request("POST", "/api/agent/bank/settle", {
+                        "operationId": operation_id, "action": action, "transfer": transfer})
+                    expected = ("committed",) if action == "commit" else ("cancelled", "rejected")
+                    if not isinstance(response, dict) or response.get("request_id") != transfer["request_id"] or response.get("state") not in expected:
+                        raise AgentError("Bank settlement was not acknowledged; runtime storage retained")
+            except (ValueError, TypeError, KeyError, AttributeError):
+                raise AgentError("Bank journal is invalid; runtime storage retained") from None
+
     def lifecycle(self, kind, instance_id, generation, payload):
         unhealthy_recovery = payload.get("unhealthyRecovery") is True
         if unhealthy_recovery and (kind not in ("instance.restart", "instance.rebuild") or payload.get("stageId") or "autoscaling" in payload):
@@ -2043,6 +2128,7 @@ class Agent:
                 if health["exists"]:
                     self.docker.call("container", "stop", "--time", "30", name)
                     self.docker.call("container", "rm", name)
+                self.settle_bank(instance_id, generation)
                 if kind == "instance.rebuild":
                     result = self.create(instance_id, next_generation, spec)
                     self.operation_phase = "removing-old-instance"
@@ -2062,7 +2148,7 @@ class Agent:
             raise AgentError("Unsupported instance operation")
         return {**self.inspect(instance_id, generation), "before": before}
 
-    def cancel_creation(self, instance_id, generation, payload):
+    def cancel_creation(self, instance_id, generation, payload, operation_id=None):
         """Explicit launch cancellation, fenced to its immutable creation identity.
 
         The operation loop serializes this with create. Once delivered, removal
@@ -2106,6 +2192,7 @@ class Agent:
                 self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, target_generation))
             if self.inspect(instance_id, target_generation)["exists"]:
                 raise AgentError("Cancelled creation container still exists")
+            self.settle_bank(instance_id, target_generation, operation_id)
             directory = self.instance_dir(instance_id, target_generation)
             if directory.exists():
                 shutil.rmtree(directory)
@@ -2747,6 +2834,7 @@ class Agent:
 
         if not unchanged_generation() or not both_missing():
             return row
+        self.settle_bank(instance_id, generation, operation["id"])
         # Preparation can leave runtime/configuration files before create's
         # checkpoint. A fresh operation must not inherit those partial files.
         expected_spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
@@ -2793,7 +2881,7 @@ class Agent:
             if kind == "instance.remove" and "autoscaling" in payload and not checkpoint:
                 return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision")
             if kind == "instance.cancel-create":
-                return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload))
+                return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload, operation["id"]))
             if kind == "instance.create":
                 marker = self.instance_dir(instance_id, generation).parent / "current.json"
                 if marker.exists():
@@ -2885,6 +2973,7 @@ class Agent:
                     directory = self.instance_dir(instance_id, observed_generation)
                     atomic_json(directory.parent / "current.json", {"generation": observed_generation})
                     if kind == "instance.rebuild" and not self.inspect(instance_id, generation)["exists"]:
+                        self.settle_bank(instance_id, generation, operation["id"])
                         old = self.instance_dir(instance_id, generation)
                         if old.exists():
                             shutil.rmtree(old)
@@ -2899,6 +2988,7 @@ class Agent:
             elif kind == "instance.remove":
                 proven = not health["exists"]
                 if proven:
+                    self.settle_bank(instance_id, generation, operation["id"])
                     directory = self.instance_dir(instance_id, generation)
                     if directory.exists():
                         shutil.rmtree(directory)

agent/tests/test_agent.py

@@ -1644,6 +1644,109 @@ class AgentTests(unittest.TestCase):
         self.assertEqual(self.agent.execute(self.operation("instance.remove"))["state"], "failed")
         self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls))
 
+    def test_create_materializes_private_bank_config_from_instance_credential(self):
+        secret = "ec_bank_" + "a" * 43
+        payload = {"name": "server", "game": "t5", "libraryId": self.library_id,
+                   "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960,
+                   "bank": {"token": secret, "configPath": "configs/electric_cherry_bank.cfg"}}
+        operation = self.operation("instance.create", payload, "create")
+        self.assertEqual(self.agent.execute(operation)["state"], "completed")
+        path = self.agent.instance_dir(INSTANCE, 1) / "app/configs/electric_cherry_bank.cfg"
+        self.assertEqual(path.read_text(), f'set tws_bank_token "{secret}"\nset tws_bank_url "https://controller.example/api/bank"\n')
+        self.assertIn("bank.settlement", self.agent.inventory()["capabilities"])
+        payload["bank"]["token"] = secret + '\nset bad "1"'
+        with self.assertRaises(AgentError):
+            self.agent.initial_files(payload)
+
+    def bank_journal(self, decisions=("commit", "cancel"), game="t5"):
+        path = self.agent.instance_dir(INSTANCE, 1) / f"app/runtime/tws-bank/{game}/scope.json"
+        reports = [json.dumps({"server": "t5_zombie_test_1", "guid": "ace", "match": "match-one", "request_id": f"{game}-{i}", "operation": "deposit", "amount": 2000, "capacity": 1000000, "decision": decision}) for i, decision in enumerate(decisions)]
+        module.atomic_json(path, {"version": 1, "server": "t5_zombie_test_1", "endpoint": "https://controller.example/api/bank", "reports": reports})
+        return path
+
+    def bank_acknowledgment(self, method, path, payload):
+        self.assertEqual((method, path), ("POST", "/api/agent/bank/settle"))
+        self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"])
+        return {"state": "committed" if payload["action"] == "commit" else "cancelled", "request_id": payload["transfer"]["request_id"]}
+
+    def test_remove_settles_both_game_journals_after_stop_before_deleting_storage(self):
+        self.create()
+        paths = [self.bank_journal(game=game) for game in ("t5", "t6")]
+        def acknowledge(*args):
+            self.assertTrue(all(path.exists() for path in paths))
+            self.assertEqual(args[2]["operationId"], "remove")
+            return self.bank_acknowledgment(*args)
+        with patch.object(self.agent, "request", side_effect=acknowledge) as request:
+            result = self.agent.execute(self.operation("instance.remove", {"safetyOverride": True}, "remove"))
+        self.assertEqual(result["state"], "completed", result)
+        self.assertEqual(request.call_count, 4)
+        self.assertTrue(all(not path.exists() for path in paths))
+
+    def test_remove_retains_bank_journal_during_outage_and_resumes_after_agent_restart(self):
+        self.create()
+        path = self.bank_journal()
+        operation = self.operation("instance.remove", {"safetyOverride": True}, "remove")
+        with patch.object(self.agent, "request", side_effect=AgentError("Controller request failed")):
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+        self.assertTrue(path.exists())
+        self.agent.journal.close()
+        self.agent = Agent(self.config, self.docker)
+        calls = list(self.docker.calls)
+        with patch.object(self.agent, "request", side_effect=self.bank_acknowledgment):
+            self.assertEqual(self.agent.execute(operation)["state"], "completed")
+        self.assertFalse(path.exists())
+        self.assertEqual(self.docker.calls, calls)
+
+    def test_rebuild_settles_before_starting_replacement_and_retains_unknown_decisions(self):
+        payload = self.create()
+        path = self.bank_journal(("commit",))
+        operation = self.operation("instance.rebuild", {"spec": payload, "nextGeneration": 2, "safetyOverride": True}, "rebuild")
+        with patch.object(self.agent, "request", return_value={"state": "committed", "request_id": "wrong"}):
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+        self.assertTrue(path.exists())
+        self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"])
+        with patch.object(self.agent, "request", side_effect=self.bank_acknowledgment):
+            # Existing rebuild recovery records the interrupted create as failed;
+            # a fresh operation can then rebuild without replaying the old command.
+            self.assertEqual(self.agent.execute(operation)["state"], "failed")
+            retry = self.operation("instance.rebuild", operation["payload"], "replacement")
+            self.assertEqual(self.agent.execute(retry)["state"], "completed")
+        self.assertFalse(path.exists())
+        self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
+
+    def test_cancel_creation_retries_bank_settlement_before_deleting_runtime(self):
+        self.create()
+        path = self.bank_journal(("cancel",))
+        operation = self.operation("instance.cancel-create", {"createOperationId": "create"}, "cancel")
+        with patch.object(self.agent, "request", side_effect=AgentError("offline")):
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+        self.assertTrue(path.exists())
+        with patch.object(self.agent, "request", side_effect=self.bank_acknowledgment):
+            self.assertEqual(self.agent.execute(operation)["state"], "completed")
+        self.assertFalse(path.exists())
+
+    def test_malformed_or_symlinked_bank_journal_blocks_storage_deletion(self):
+        self.create()
+        path = self.bank_journal()
+        path.write_text('{broken')
+        operation = self.operation("instance.remove", {"safetyOverride": True}, "remove")
+        with patch.object(self.agent, "request") as request:
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+            self.assertTrue(path.exists())
+            outside = Path(self.temp.name) / "outside.json"
+            outside.write_text('{"reports": []}')
+            self.bank_journal()
+            journal = json.loads(path.read_text())
+            journal["endpoint"] = "https://untrusted.example/api/bank"
+            path.write_text(json.dumps(journal))
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+            path.unlink()
+            path.symlink_to(outside)
+            self.assertEqual(self.agent.execute(operation)["state"], "unknown")
+            self.assertTrue(path.is_symlink())
+            request.assert_not_called()
+
     def test_rebuild_moves_generation_and_rejects_old_commands(self):
         payload = self.create()
         self.agent.rcon = lambda *_: {"ok": True, "players": 0}

docker-compose.yml

@@ -15,7 +15,6 @@ services:
             ORIGIN: ${EC_PUBLIC_URL:-http://localhost:3000}
             EC_SETUP_TOKEN: ${EC_SETUP_TOKEN:-}
             EC_TELEMETRY_TOKEN: ${EC_TELEMETRY_TOKEN:-}
-            EC_BANK_TOKENS: ${EC_BANK_TOKENS:-}
             EC_BANK_MAX_BALANCE: ${EC_BANK_MAX_BALANCE:-250000}
             EC_AUTH_ALLOW_HTTP: ${EC_AUTH_ALLOW_HTTP:-false}
             # Legacy host imports stream whole game files; buffering routes enforce their own caps.

docs/guides/bank.md

@@ -9,32 +9,45 @@ IDs. This release does not introduce separate bank-assisted rankings.
 
 ## Setup and rollout
 
-1. Back up the controller database. Deploy Electric Cherry first and restart
+1. Back up the controller database. Update every host agent to **1.9.0 or later**
-   the controller. Migration 20 creates accounts, transfer requests and a ledger.
+   before enabling the bank. This version settles game journals before removing
-2. Generate a **different** secret for each registered game server, for example
+   runtime storage; older agents do not. Package and offline-sign the agent
-   `ec_bank_` followed by `openssl rand -hex 32`. Configure `EC_BANK_TOKENS` as a
+   release using the [host update workflow](native-hosts.md).
-   JSON object mapping exact server names to these secrets. Docker Compose
+2. Deploy Electric Cherry and restart the controller. Migration 20 creates
-   forwards this environment variable. An example shape with placeholders is
+   accounts, requests and the ledger; migration 21 adds instance credentials and
-   `{"t5_zombie_theater_1":"ec_bank_<random-secret>"}`.
+   binds existing requests to their currently active native instance, if any.
 3. Set `EC_BANK_MAX_BALANCE` (default 250000, range 1–1000000). Invalid configuration
-   refuses new deposits. A limit reduction does not confiscate existing balances
+   refuses new deposits. Lowering the limit preserves existing balances and
-   or invalidate already prepared deposits; withdrawals remain available.
+   prepared deposits; withdrawals remain available.
 4. Build/install TWS plugins `t5-plugin`/`t6-plugin` 1.1.0 or later and restart
    the **whole game-server process**. Install `tws_t5_bank.gsc` or
-   `tws_t6_bank.gsc` 1.0.0, the game's common module (1.0.0+) and telemetry
+   `tws_t6_bank.gsc` 1.0.0, common 1.0.0+ and telemetry 1.0.0+.
-   reporter (1.0.0+). Restart the map after installing scripts/configuration.
+5. Create or rebuild the servers through EC. Launch preparation generates a
-5. In each server's private configuration, set `tws_server_id` to its registered
+   cryptographically random `ec_bank_` credential for each immutable instance,
-   name and `tws_bank_token` to that server's secret. The optional `tws_bank_url`
+   stores it in the private controller database, and injects it through
-   defaults to `https://ec.thewrightserver.net/api/bank`; overrides must use
+   `configs/electric_cherry_bank.cfg`. The launch command executes this private
-   HTTPS. Keep these settings out of shared scripts and public config downloads.
+   file and sets `tws_server_id`. The host agent writes the private file and derives `tws_bank_url` from its
+   enrolled controller URL. Use an HTTPS controller URL.
 
-Bank tokens must match `ec_bank_` followed by 32–128 URL-safe letters, digits,
+There is **no `EC_BANK_TOKENS` environment map** and no per-server secret setup.
-underscores or hyphens. Missing, malformed, duplicate, or telemetry-reused tokens
+The private configuration is sent only in that host's authenticated operation
-disable authentication. `EC_TELEMETRY_TOKEN`, host tokens and browser sessions
+payload. It is excluded from shared recipes, promoted files and blob downloads.
-cannot authorize bank requests. The credential determines the server, and a
+Do not upload `electric_cherry_bank.cfg`: its filename is reserved. Protect the
-different `server` in the body is refused. Rotate a token by updating both the
+controller database, host operation journal and per-instance runtime directory,
-controller mapping and that server's private configuration, then restart them.
+which contain private credentials.
-Keep the same server name and bank URL so existing decisions can finish.
+
+Retries, rebuilds and host moves of the same instance reuse its credential.
+Retiring the instance immediately revokes authentication. Creating another
+server with the same display name produces a new instance ID and secret; it
+cannot finish the old instance's requests. Accounts remain attached to player
+GUIDs, so balances survive the servers. Telemetry tokens and browser sessions
+cannot authorize bank requests. Host credentials can only settle decisions
+through the separate, operation-scoped teardown endpoint.
+
+For an upgrade from the initial static-token implementation, drain outstanding
+transfers while the old controller/configuration still runs, then deploy and
+rebuild. Remove the obsolete environment map and any manually supplied token.
+Do not replace identities or discard pending journals to force an upgrade.
 
 ## Machine API
 
@@ -76,7 +89,8 @@ Preparation requires the GUID to be present in the server's latest active match,
 with the exact telemetry match key. Late joiners may need to wait for the next
 30-second heartbeat. Unknown/destroyed servers, absent players and old matches
 cannot prepare transfers. Settlement of an existing decision remains possible
-after match completion, retention or server deletion.
+after match completion or retention. Host teardown can finish saved decisions
+after retirement; the retired game credential itself no longer authenticates.
 
 Successful application responses include `ok`, `state`, `code`, `request_id`,
 `amount` and `balance`. States are `balance`, `prepared`, `committed`, `cancelled`
@@ -110,7 +124,8 @@ decision, retries only finish the ledger; they never change game points again.
 These points now belong to the current match, so an applied withdrawal is not
 automatically refunded if that match ends or its server crashes.
 
-Keep `fs_homepath/tws-bank/t5/` or `t6/` on persistent writable storage with a
+Keep `tws-bank/t5/` or `tws-bank/t6/`, relative to the game process working
+directory, on persistent writable storage with a
 single process per server/endpoint. Journals contain transfer data, not secrets.
 On a map/process restart, saved decisions settle before new bank commands.
 Unreadable storage disables the bank; failed writes prevent points changes.
@@ -121,6 +136,18 @@ original journal to resolve holds; never delete a pending journal to unblock
 the bank. If storage is permanently lost, reconcile against the retained
 request and ledger records before any manual repair.
 
+During removal, rebuild, host migration or launch cancellation, agent 1.9.0+
+stops the game and drains its journals **before** deleting runtime files or
+starting a replacement. It uses its host credential at
+`POST /api/agent/bank/settle`; EC checks the assigned host, immutable instance,
+unfinished teardown operation, server and original request identity. This
+endpoint permits only commit/cancel, never new deposits or withdrawals. Journal
+URLs must match that host's configured controller; the agent never follows a
+journal URL. Corruption, a mismatched controller, failed acknowledgment or an
+outage retains storage and leaves the operation unresolved for retry. An
+interrupted rebuild uses the existing rebuild recovery workflow to retry with
+a fresh operation after settlement. Do not manually erase its old directory.
+
 The journal provides recovery from map/process restart. It cannot make live
 game memory and SQLite one atomic transaction or preserve match points after a
 machine/storage failure. Back up persistent storage and verify crash behavior

docs/guides/native-hosts.md

@@ -1035,3 +1035,12 @@ game hosts. Run:
 - Controller restart, agent restart and network loss before execution, during
   execution and after execution but before acknowledgement. Verify the same
   operation IDs and absence of duplicate containers/restarts.
+
+### Bank journals during server teardown
+
+Agent 1.9.0 adds `bank.settlement`: removal, rebuild, migration and creation
+cancellation finish pending TWS bank decisions before deleting the old runtime.
+An unavailable controller or corrupt journal leaves the operation unresolved
+and preserves storage. Update all agents before enabling the
+[player bank](bank.md). EC generates and injects bank credentials per instance;
+no token map or manual server-secret provisioning is required.

src/lib/server/auth/bank.ts

@@ -1,41 +1,46 @@
-import { createHash, timingSafeEqual } from 'node:crypto';
+import { createHash, randomBytes } from 'node:crypto';
+import { db } from '../db/index.js';
 
-const digest = (value: string) => createHash('sha256').update(value).digest();
+const digest = (value: string) => createHash('sha256').update(value).digest('hex');
-const valid = (value: unknown): value is string =>
-    typeof value === 'string' && /^ec_bank_[A-Za-z0-9_-]{32,128}$/.test(value);
 
-/** Dedicated, unique credentials; telemetry and browser credentials never authorize spending. */
+/** Private controller state, reused only by the same immutable native instance. */
+export function bankCredential(instanceId: string): string {
+    return db()
+        .transaction(() => {
+            const instance = db()
+                .prepare('SELECT server FROM native_instances WHERE id=? AND retired_at IS NULL')
+                .get(instanceId) as { server: string } | undefined;
+            if (!instance) throw new Error('Bank credentials require an active native instance.');
+            const saved = db()
+                .prepare('SELECT token FROM bank_credentials WHERE instance_id=?')
+                .get(instanceId) as { token: string } | undefined;
+            if (saved) return saved.token;
+            const token = 'ec_bank_' + randomBytes(32).toString('base64url');
+            db()
+                .prepare(
+                    'INSERT INTO bank_credentials(instance_id,server,token,token_hash,created_at) VALUES (?,?,?,?,?)'
+                )
+                .run(instanceId, instance.server, token, digest(token), new Date().toISOString());
+            return token;
+        })
+        .immediate();
+}
+
+/** Retiring an instance immediately revokes its game credential, including reads. */
 export function authenticateBank(
     request: Request
-): { ok: true; server: string } | { ok: false; status: 401 | 503; code: string } {
+): { ok: true; server: string; instanceId: string } | { ok: false; status: 401; code: string } {
-    let entries: [string, string][];
-    try {
-        const config: unknown = JSON.parse(process.env.EC_BANK_TOKENS ?? '{}');
-        if (!config || typeof config !== 'object' || Array.isArray(config)) throw new Error();
-        const values = Object.entries(config);
-        if (
-            !values.length ||
-            values.some(
-                ([server, token]) =>
-                    !server || !valid(token) || token === process.env.EC_TELEMETRY_TOKEN
-            )
-        )
-            throw new Error();
-        entries = values as [string, string][];
-        if (new Set(entries.map(([, token]) => token)).size !== entries.length) throw new Error();
-    } catch {
-        return { ok: false, status: 503, code: 'unconfigured' };
-    }
     const token = request.headers
         .get('authorization')
         ?.match(/^Bearer (ec_bank_[A-Za-z0-9_-]{32,128})$/i)?.[1];
-    if (!token) return { ok: false, status: 401, code: 'unauthorized' };
+    const matched = token
-    const supplied = digest(token);
+        ? (db()
-    let matched: string | undefined;
+              .prepare(
-    for (const [server, expected] of entries) {
+                  `SELECT c.server, c.instance_id AS instanceId
-        if (timingSafeEqual(supplied, digest(expected))) matched = server;
+        FROM bank_credentials c JOIN native_instances n ON n.id=c.instance_id AND n.server=c.server
-    }
+        WHERE c.token_hash=? AND n.retired_at IS NULL`
-    return matched
+              )
-        ? { ok: true, server: matched }
+              .get(digest(token)) as { server: string; instanceId: string } | undefined)
-        : { ok: false, status: 401, code: 'unauthorized' };
+        : undefined;
+    return matched ? { ok: true, ...matched } : { ok: false, status: 401, code: 'unauthorized' };
 }

src/lib/server/db/bank-credentials.ts

@@ -0,0 +1,22 @@
+import type Database from 'better-sqlite3';
+
+export function migrateBankCredentials(handle: Database.Database): void {
+    handle.exec(`
+        CREATE TABLE IF NOT EXISTS bank_credentials (
+            instance_id TEXT PRIMARY KEY, server TEXT NOT NULL,
+            token TEXT NOT NULL, token_hash TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL
+        );
+    `);
+    if (
+        !(handle.pragma('table_info(bank_requests)') as { name: string }[]).some(
+            (column) => column.name === 'instance_id'
+        )
+    ) {
+        handle.exec(`
+            ALTER TABLE bank_requests ADD COLUMN instance_id TEXT;
+            UPDATE bank_requests SET instance_id = (
+                SELECT id FROM native_instances WHERE server = bank_requests.server AND retired_at IS NULL
+            );
+        `);
+    }
+}

src/lib/server/db/migrations.ts

@@ -19,6 +19,7 @@ import { migrateGameFileSchema } from '../game-files/schema.js';
 import { migrateLeaderboardReports } from './leaderboard-reports.js';
 import { migratePlayerScoreTotal } from './player-score-total.js';
 import { migrateBank } from './bank.js';
+import { migrateBankCredentials } from './bank-credentials.js';
 import type { Migration } from './migration-runner.js';
 
 // Append only: never renumber entries or change an applied migration to add an
@@ -111,7 +112,8 @@ export const migrations: readonly Migration[] = [
         name: 'cumulative player score per match',
         run: migratePlayerScoreTotal
     },
-    { version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank }
+    { version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank },
+    { version: 21, name: 'instance-owned bank credentials', run: migrateBankCredentials }
 ];
 
 export const databaseVersion = migrations.length;

src/lib/server/fleet/bank.ts

@@ -20,6 +20,7 @@ interface Result {
     balance: number;
 }
 interface Saved {
+    instance_id: string | null;
     fingerprint: string;
     state: string;
     amount: number;
@@ -79,7 +80,8 @@ export function bankProfile(guid: string) {
 export function bankRequest(
     action: BankAction,
     value: unknown,
-    authenticatedServer: string
+    authenticatedServer: string,
+    instanceId: string
 ): { status: number; body: Result } {
     const result = (status: number, code: string, request_id = '') => ({
         status,
@@ -121,7 +123,7 @@ export function bankRequest(
             const saved = handle
                 .prepare('SELECT * FROM bank_requests WHERE server = ? AND request_id = ?')
                 .get(server, request_id) as Saved | undefined;
-            if (saved && saved.fingerprint !== fingerprint)
+            if (saved && (saved.instance_id !== instanceId || saved.fingerprint !== fingerprint))
                 return result(409, 'request_conflict', request_id);
             const balance =
                 (
@@ -145,8 +147,8 @@ export function bankRequest(
             const insert = (body: Result, matchId: number | null = null) => {
                 handle
                     .prepare(
-                        `INSERT INTO bank_requests(server,request_id,guid,fingerprint,operation,match_id,amount,state,result,created_at)
+                        `INSERT INTO bank_requests(server,request_id,guid,fingerprint,operation,match_id,amount,state,result,created_at,instance_id)
-                VALUES (?,?,?,?,?,?,?,?,?,?)`
+                VALUES (?,?,?,?,?,?,?,?,?,?,?)`
                     )
                     .run(
                         server,
@@ -158,7 +160,8 @@ export function bankRequest(
                         body.amount,
                         body.state,
                         JSON.stringify(body),
-                        now
+                        now,
+                        instanceId
                     );
                 return { status: 200, body };
             };
@@ -200,6 +203,14 @@ export function bankRequest(
                 return { status: 200, body };
             }
             if (saved) return { status: 200, body: JSON.parse(saved.result) as Result };
+            if (
+                !handle
+                    .prepare(
+                        'SELECT 1 FROM native_instances WHERE id=? AND server=? AND retired_at IS NULL'
+                    )
+                    .get(instanceId, server)
+            )
+                return result(403, 'retired_instance', request_id);
             const member = handle
                 .prepare(
                     `SELECT m.id FROM leaderboard_matches m

src/lib/server/provisioning/launch-preparation.ts

@@ -13,6 +13,7 @@ export interface NativeLaunchInput {
     instance: NativeInstance;
     serverKey: string;
     rconPassword: string;
+    bankToken: string;
     rebuild: boolean;
     safetyOverride?: boolean;
     unhealthyRecovery?: boolean;

src/lib/server/provisioning/launch-worker.ts

@@ -71,6 +71,19 @@ parentPort!.on(
                 };
             })();
             const instance = input.instance;
+            // Credential files travel only in this host's private operation payload, never
+            // shared recipes, promoted files, or the content-addressed download store.
+            const bankPath = posix.join(
+                relDestForKind('config', recipe.game),
+                'electric_cherry_bank.cfg'
+            );
+            if (
+                recipe.files.some(
+                    (file) =>
+                        posix.join(relDestForKind(file.kind, recipe.game), file.path) === bankPath
+                )
+            )
+                throw new Error('electric_cherry_bank.cfg is reserved for instance credentials.');
             const payload = {
                 name: instance.server,
                 game: recipe.game,
@@ -79,13 +92,14 @@ parentPort!.on(
                 port: instance.port,
                 bindAddress: instance.bind_address,
                 rconPassword: password,
+                bank: { token: input.bankToken, configPath: bankPath },
                 env: {
                     PLUTAINER_SERVER_NAME: instance.server,
                     PLUTAINER_PORT: String(instance.port),
                     PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp',
                     PLUTAINER_CONFIG_FILE: recipe.configFile,
                     PLUTAINER_MAP_ROTATE: 'false',
-                    PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`,
+                    PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +exec electric_cherry_bank.cfg +set tws_server_id ${instance.server} +map_rotate`,
                     PLUTO_SERVER_KEY: input.serverKey
                 },
                 files: recipe.files.map((file) => ({

src/lib/server/provisioning/native.ts

@@ -12,6 +12,7 @@ import { isIP } from 'node:net';
 import { setImmediate as yieldToController } from 'node:timers/promises';
 import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
 import { randomBytes, randomUUID } from 'node:crypto';
+import { bankCredential } from '../auth/bank.js';
 import { posix } from 'node:path';
 import { db } from '../db/index.js';
 import { unsettledCancelledCommand } from './cancellation.js';
@@ -1936,6 +1937,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 instance,
                 serverKey: preparedKey,
                 rconPassword: preparedPassword,
+                bankToken: bankCredential(instance.id),
                 rebuild: job.action === 'rebuild' && !migration,
                 safetyOverride: overrideSafety,
                 unhealthyRecovery: host.inventory.capabilities?.includes(

src/routes/api/agent/bank/settle/+server.ts

@@ -0,0 +1,28 @@
+import { error, json } from '@sveltejs/kit';
+import { authenticateAgent, objectBody } from '$lib/server/hosts/http.js';
+import { db } from '$lib/server/db/index.js';
+import { bankRequest } from '$lib/server/fleet/bank.js';
+
+/** An assigned teardown may finish durable decisions, never prepare new spending. */
+export async function POST({ request }: { request: Request }) {
+    const host = authenticateAgent(request);
+    const body = await objectBody(request, 8192);
+    if (typeof body.operationId !== 'string' || !['commit', 'cancel'].includes(String(body.action)))
+        error(400, 'Invalid bank settlement.');
+    const scope = db()
+        .prepare(
+            `SELECT n.id, n.server FROM ec_host_operations o
+        JOIN native_instances n ON n.id=o.instance_id
+        WHERE o.id=? AND o.host_id=? AND o.kind IN ('instance.remove','instance.rebuild','instance.cancel-create')
+        AND o.state IN ('pending','accepted','running','unknown')`
+        )
+        .get(body.operationId, host.id) as { id: string; server: string } | undefined;
+    if (!scope) error(403, 'Bank settlement requires an assigned, unfinished teardown.');
+    const result = bankRequest(
+        body.action as 'commit' | 'cancel',
+        body.transfer,
+        scope.server,
+        scope.id
+    );
+    return json(result.body, { status: result.status, headers: { 'Cache-Control': 'no-store' } });
+}

src/routes/api/bank/[action]/+server.ts

@@ -29,7 +29,8 @@ export async function POST({ request, params }: { request: Request; params: { ac
         const { body, status } = bankRequest(
             params.action as BankAction,
             value,
-            authentication.server
+            authentication.server,
+            authentication.instanceId
         );
         return json(body, { status, headers });
     } catch {

static/agent/ec-agent.py

@@ -35,7 +35,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.8.7"
+VERSION = "1.9.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 MANIFEST_MAX_ENTRIES = 10000
@@ -1503,7 +1503,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         plutonium_cache = self.plutonium_cache_inventory()
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
@@ -1657,7 +1657,7 @@ class Agent:
         # record. Whenever preparation was recorded, enforce that exact image.
         if image in prepared and prepared[image] != image_id:
             raise AgentError("Prepared runtime image has changed. An admin must prepare it again before launching.")
-        files = payload.get("files", [])
+        files = self.initial_files(payload)
         if not isinstance(files, list) or len(files) > 10000:
             raise Rejected("Invalid initial file list")
         total, paths = 0, set()
@@ -1911,7 +1911,7 @@ class Agent:
         self.operation_phase = "seeding-runtime"
         self.seed_plutonium(app, image)
         self.operation_phase = "writing-create-files"
-        self.write_initial_files(app, payload.get("files", []))
+        self.write_initial_files(app, self.initial_files(payload))
         self.operation_phase = "persisting-create-metadata"
         atomic_json(directory / "spec.json", payload)
         env_path = directory / "environment"
@@ -1944,6 +1944,25 @@ class Agent:
         if self.executing_id and self.executing_id in self.cancelled_creations:
             raise AgentError("Creation cancelled; waiting for instance cleanup")
 
+    def initial_files(self, payload):
+        files = payload.get("files", [])
+        if not isinstance(files, list):
+            raise Rejected("Invalid initial file list")
+        bank = payload.get("bank")
+        if bank is None:
+            return files
+        if not isinstance(bank, dict) or not isinstance(bank.get("token"), str) or not re.fullmatch(r"ec_bank_[A-Za-z0-9_-]{32,128}", bank["token"]):
+            raise Rejected("Invalid instance bank credential")
+        path = bank.get("configPath")
+        scoped(self.root / "agent" / "validation", path)
+        if PurePosixPath(path).name != "electric_cherry_bank.cfg":
+            raise Rejected("Invalid private bank config path")
+        controller = self.config["controller"].rstrip("/")
+        if any(char in controller for char in ('"', "\\", "\n", "\r", "\x00")):
+            raise Rejected("Invalid bank controller URL")
+        content = f'set tws_bank_token "{bank["token"]}"\nset tws_bank_url "{controller}/api/bank"\n'.encode()
+        return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}]
+
     def write_initial_files(self, root, files):
         if not isinstance(files, list) or len(files) > 10000:
             raise AgentError("Invalid file list")
@@ -1992,6 +2011,72 @@ class Agent:
         if (time.monotonic() - started) * 1000 >= ttl:
             raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation")
 
+    def settle_bank(self, instance_id, generation, operation_id=None):
+        """Drain stopped runtime journals before discarding any generation's storage.
+
+        Use only our configured controller and host credential. Never follow an
+        endpoint or symlink supplied by a game journal. Lost replies are retried
+        with the original request identity; files remain until teardown succeeds.
+        """
+        app = scoped(self.instance_dir(instance_id, generation), "app")
+        if not app.exists():
+            return
+        if self.inspect(instance_id, generation)["running"]:
+            raise AgentError("Bank settlement requires a stopped instance")
+        operation_id = operation_id or self.executing_id
+        visited = 0
+        deadline = time.monotonic() + 30
+        journals = []
+
+        def unreadable(_error):
+            raise AgentError("Bank journal discovery failed; runtime storage retained")
+
+        for directory, directories, files in os.walk(app, followlinks=False, onerror=unreadable):
+            relative = Path(directory).relative_to(app)
+            inside_bank = "tws-bank" in relative.parts
+            visited += 1
+            if visited > 100000 or time.monotonic() > deadline:
+                raise AgentError("Bank journal discovery limit exceeded; runtime storage retained")
+            for name in directories + files:
+                if (inside_bank or name == "tws-bank") and (Path(directory) / name).is_symlink():
+                    raise AgentError("Bank journal paths must not be symlinks")
+            if not inside_bank:
+                continue
+            for name in files:
+                if name.endswith(".json"):
+                    journals.append((relative / name).as_posix())
+                    if len(journals) > 4096:
+                        raise AgentError("Too many bank journals; runtime storage retained")
+        for path in sorted(journals):
+            try:
+                raw = managed_read(app, path)
+            except Rejected:
+                raise AgentError("Bank journal cannot be read; runtime storage retained") from None
+            if raw is None or len(raw) > 16 * 1024 * 1024:
+                raise AgentError("Bank journal is unavailable or oversized")
+            try:
+                journal = json.loads(raw)
+                reports = journal["reports"]
+                if journal["endpoint"].rstrip("/") != self.config["controller"].rstrip("/") + "/api/bank":
+                    raise AgentError("Bank journal belongs to a different controller; runtime storage retained")
+                if type(journal["version"]) is not int or journal["version"] != 1 or not isinstance(reports, list) or len(reports) > 4096:
+                    raise ValueError()
+                # Controller independently binds this operation to the instance/server.
+                for report in reports:
+                    if not isinstance(report, str) or len(report.encode()) > 20000:
+                        raise ValueError()
+                    transfer = json.loads(report)
+                    action = transfer["decision"]
+                    if action not in ("commit", "cancel") or transfer["server"] != journal["server"] or not operation_id:
+                        raise ValueError()
+                    response = self.request("POST", "/api/agent/bank/settle", {
+                        "operationId": operation_id, "action": action, "transfer": transfer})
+                    expected = ("committed",) if action == "commit" else ("cancelled", "rejected")
+                    if not isinstance(response, dict) or response.get("request_id") != transfer["request_id"] or response.get("state") not in expected:
+                        raise AgentError("Bank settlement was not acknowledged; runtime storage retained")
+            except (ValueError, TypeError, KeyError, AttributeError):
+                raise AgentError("Bank journal is invalid; runtime storage retained") from None
+
     def lifecycle(self, kind, instance_id, generation, payload):
         unhealthy_recovery = payload.get("unhealthyRecovery") is True
         if unhealthy_recovery and (kind not in ("instance.restart", "instance.rebuild") or payload.get("stageId") or "autoscaling" in payload):
@@ -2043,6 +2128,7 @@ class Agent:
                 if health["exists"]:
                     self.docker.call("container", "stop", "--time", "30", name)
                     self.docker.call("container", "rm", name)
+                self.settle_bank(instance_id, generation)
                 if kind == "instance.rebuild":
                     result = self.create(instance_id, next_generation, spec)
                     self.operation_phase = "removing-old-instance"
@@ -2062,7 +2148,7 @@ class Agent:
             raise AgentError("Unsupported instance operation")
         return {**self.inspect(instance_id, generation), "before": before}
 
-    def cancel_creation(self, instance_id, generation, payload):
+    def cancel_creation(self, instance_id, generation, payload, operation_id=None):
         """Explicit launch cancellation, fenced to its immutable creation identity.
 
         The operation loop serializes this with create. Once delivered, removal
@@ -2106,6 +2192,7 @@ class Agent:
                 self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, target_generation))
             if self.inspect(instance_id, target_generation)["exists"]:
                 raise AgentError("Cancelled creation container still exists")
+            self.settle_bank(instance_id, target_generation, operation_id)
             directory = self.instance_dir(instance_id, target_generation)
             if directory.exists():
                 shutil.rmtree(directory)
@@ -2747,6 +2834,7 @@ class Agent:
 
         if not unchanged_generation() or not both_missing():
             return row
+        self.settle_bank(instance_id, generation, operation["id"])
         # Preparation can leave runtime/configuration files before create's
         # checkpoint. A fresh operation must not inherit those partial files.
         expected_spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
@@ -2793,7 +2881,7 @@ class Agent:
             if kind == "instance.remove" and "autoscaling" in payload and not checkpoint:
                 return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision")
             if kind == "instance.cancel-create":
-                return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload))
+                return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload, operation["id"]))
             if kind == "instance.create":
                 marker = self.instance_dir(instance_id, generation).parent / "current.json"
                 if marker.exists():
@@ -2885,6 +2973,7 @@ class Agent:
                     directory = self.instance_dir(instance_id, observed_generation)
                     atomic_json(directory.parent / "current.json", {"generation": observed_generation})
                     if kind == "instance.rebuild" and not self.inspect(instance_id, generation)["exists"]:
+                        self.settle_bank(instance_id, generation, operation["id"])
                         old = self.instance_dir(instance_id, generation)
                         if old.exists():
                             shutil.rmtree(old)
@@ -2899,6 +2988,7 @@ class Agent:
             elif kind == "instance.remove":
                 proven = not health["exists"]
                 if proven:
+                    self.settle_bank(instance_id, generation, operation["id"])
                     directory = self.instance_dir(instance_id, generation)
                     if directory.exists():
                         shutil.rmtree(directory)

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.8.7",
+  "version": "1.9.0",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "e773c907141c97eab2055b354306973b248a67109a51337f1f1d76d80aaa8701",
+  "sha256": "07f91ba8780c1e2b0a9c0ab3534c1731d4246d3f5d84fece14691c930dcb5e90",
   "updaterProtocolVersion": 2,
   "updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb",
   "installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231"

static/agent/manifest.json.sig

@@ -1,8 +0,0 @@
------BEGIN PGP SIGNATURE-----
-
-iJEEABYKADkWIQSpocb6VBfQzvdIh9HF6fdSNniN+gUCasmr1RsUgAAAAAAEAA5t
-YW51MiwyLjUrMS4xMiwwLDMACgkQxen3UjZ4jfprZQD+PPqrtEYWTImhdy1eFAfv
-k5XD1IaFOgSvcsL9P0tj69IBAIRGt3zqWw9E8vFeyZXz58GfnscyFI6ws5ln+l3h
-i2UP
-=Er1r
------END PGP SIGNATURE-----

tests/unit/bank.test.ts

@@ -1,3 +1,4 @@
+import { randomUUID } from 'node:crypto';
 import { afterEach, beforeEach, expect, it, vi } from 'vitest';
 import { newFixture, type Fixture } from '../helpers/fixture.js';
 import { routePermission } from '../../src/lib/server/auth/access.js';
@@ -5,20 +6,38 @@ import { routePermission } from '../../src/lib/server/auth/access.js';
 let fixture: Fixture;
 const server = 't5_zombie_theater_1';
 const other = 't6_zm_transit_1';
-const secret = 'ec_bank_' + 'a'.repeat(40);
+let secret: string;
-const otherSecret = 'ec_bank_' + 'b'.repeat(40);
+let otherSecret: string;
+let instanceId: string;
+let hostId: string;
+let hostToken: string;
 let serial = 0;
 
 beforeEach(async () => {
     vi.resetModules();
     fixture = await newFixture([server, other]);
-    vi.stubEnv('EC_BANK_TOKENS', JSON.stringify({ [server]: secret, [other]: otherSecret }));
     vi.stubEnv('EC_TELEMETRY_TOKEN', 'telemetry_' + 't'.repeat(40));
     vi.stubEnv('EC_BANK_MAX_BALANCE', '250000');
     const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
     const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
     await seedServerRegistry();
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const { bankCredential } = await import('../../src/lib/server/auth/bank.js');
+    const { createHost, createEnrollment, enrollHost } =
+        await import('../../src/lib/server/hosts/index.js');
+    hostId = createHost({ name: 'Bank host' }).id;
+    hostToken = enrollHost(createEnrollment(hostId).token, {}).credential;
     for (const source of [server, other]) {
+        const id = randomUUID();
+        db()
+            .prepare(
+                `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES (?,?,?,'library','image',28960,?)`
+            )
+            .run(id, source, hostId, new Date().toISOString());
+        if (source === server) {
+            instanceId = id;
+            secret = bankCredential(id);
+        } else otherSecret = bankCredential(id);
         saveTelemetry({
             server: source,
             map: source === server ? 'zombie_theater' : 'zm_transit',
@@ -84,21 +103,88 @@ it('routes only bank POSTs as machine requests, authenticating before reading bo
     });
 });
 
-it('fails closed for missing, duplicate, telemetry-reused and invalid credentials', async () => {
+it('owns generated secrets by immutable instance, preserving rebuilds and revoking retired names', async () => {
-    for (const configured of [
+    const { db, closeDb } = await import('../../src/lib/server/db/index.js');
-        '',
+    const { bankCredential } = await import('../../src/lib/server/auth/bank.js');
-        '{}',
+    const data = transfer();
-        '[]',
+    await post('deposit', data);
-        '{',
+    expect(secret).toMatch(/^ec_bank_[A-Za-z0-9_-]{43}$/);
-        JSON.stringify({ [server]: secret, [other]: secret }),
+    expect(otherSecret).not.toBe(secret);
-        JSON.stringify({ [server]: 'short' })
+    closeDb();
-    ]) {
+    db().prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId);
-        vi.stubEnv('EC_BANK_TOKENS', configured);
+    expect(bankCredential(instanceId)).toBe(secret);
-        expect(await post('balance', { server, guid: 'ace' })).toMatchObject({ status: 503 });
+    db()
-    }
+        .prepare('UPDATE native_instances SET retired_at=? WHERE id=?')
-    vi.stubEnv('EC_BANK_TOKENS', JSON.stringify({ [server]: secret }));
+        .run(new Date().toISOString(), instanceId);
-    vi.stubEnv('EC_TELEMETRY_TOKEN', secret);
+    expect(await post('balance', { server, guid: 'ace' })).toMatchObject({ status: 401 });
-    expect(await post('balance', { server, guid: 'ace' })).toMatchObject({ status: 503 });
+    expect(() => bankCredential(instanceId)).toThrow('active native instance');
+    const next = randomUUID();
+    db()
+        .prepare(
+            `INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES (?,?,?,'library','image',28960,?)`
+        )
+        .run(next, server, hostId, new Date().toISOString());
+    const replacement = bankCredential(next);
+    const { migrateBankCredentials } = await import('../../src/lib/server/db/bank-credentials.js');
+    migrateBankCredentials(db()); // A legacy migration rerun must not rebind old requests.
+    expect(replacement).not.toBe(secret);
+    expect(await post('commit', data, replacement)).toMatchObject({
+        status: 409,
+        code: 'request_conflict'
+    });
+    expect(await post('cancel', data, replacement)).toMatchObject({ status: 409 });
+});
+
+it('settles journals only for the host assigned to an unfinished teardown, even after retirement', async () => {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const { queueHostOperation } = await import('../../src/lib/server/hosts/index.js');
+    const { POST } = await import('../../src/routes/api/agent/bank/settle/+server.js');
+    const data = transfer();
+    await post('deposit', data);
+    const removal = queueHostOperation(
+        hostId,
+        'instance.remove',
+        {},
+        { instanceId, generation: 1 }
+    );
+    db()
+        .prepare('UPDATE native_instances SET retired_at=? WHERE id=?')
+        .run(new Date().toISOString(), instanceId);
+    const settle = (
+        action = 'commit',
+        operationId = removal.id,
+        credential = hostToken,
+        value = data
+    ) =>
+        POST({
+            request: new Request('https://test/api/agent/bank/settle', {
+                method: 'POST',
+                headers: { Authorization: `Bearer ${credential}` },
+                body: JSON.stringify({ operationId, action, transfer: value })
+            })
+        });
+    await expect(settle('commit', removal.id, secret)).rejects.toMatchObject({ status: 401 });
+    const { createHost, createEnrollment, enrollHost } =
+        await import('../../src/lib/server/hosts/index.js');
+    const stranger = createHost({ name: 'Unassigned host' });
+    const strangerToken = enrollHost(createEnrollment(stranger.id).token, {}).credential;
+    await expect(settle('commit', removal.id, strangerToken)).rejects.toMatchObject({
+        status: 403
+    });
+    await expect(settle('deposit')).rejects.toMatchObject({ status: 400 });
+    await expect(settle('commit', randomUUID())).rejects.toMatchObject({ status: 403 });
+    expect(
+        await (await settle('commit', removal.id, hostToken, { ...data, server: other })).json()
+    ).toMatchObject({ code: 'wrong_server' });
+    const first = await (await settle()).json();
+    expect(first).toMatchObject({ state: 'committed', balance: 2000 });
+    expect(await (await settle()).json()).toEqual(first);
+    const cancelled = transfer();
+    expect(await (await settle('cancel', removal.id, hostToken, cancelled)).json()).toMatchObject({
+        state: 'cancelled'
+    });
+    db().prepare("UPDATE ec_host_operations SET state='completed' WHERE id=?").run(removal.id);
+    await expect(settle()).rejects.toMatchObject({ status: 403 });
 });
 
 it('persists one credit and one ledger row through lost replies, retries and database reopen', async () => {

tests/unit/native-launch-preparation.test.ts

@@ -416,3 +416,47 @@ describe('asynchronous native launch preparation', () => {
         expect(await operationFor(second.deploymentId)).toBeDefined();
     });
 });
+
+it('injects unique persistent bank credentials only into private runtime files', async () => {
+    const job = await queue();
+    await runPass();
+    const op = await operationFor(job.deploymentId);
+    const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+    const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+    const { bankCredential, authenticateBank } = await import('../../src/lib/server/auth/bank.js');
+    const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
+    const instance = nativeInstance(job.name)!;
+    const credential = bankCredential(instance.id);
+    const payload = getHostOperation(op!.id)!.payload as {
+        bank: { token: string; configPath: string };
+        files: { path: string }[];
+        env: Record<string, string>;
+    };
+    expect(payload.bank).toEqual({
+        token: credential,
+        configPath: 'configs/electric_cherry_bank.cfg'
+    });
+    expect(payload.files.some((file) => file.path.endsWith('electric_cherry_bank.cfg'))).toBe(
+        false
+    );
+    const { db } = await import('../../src/lib/server/db/index.js');
+    expect(
+        db()
+            .prepare('SELECT count(*) AS n FROM file_blobs WHERE instr(content,?) > 0')
+            .get(Buffer.from(credential))
+    ).toEqual({ n: 0 });
+    expect(payload.env.PLUTAINER_EXTRA_ARGS).toContain(
+        '+exec electric_cherry_bank.cfg +set tws_server_id'
+    );
+    expect(
+        authenticateBank(
+            new Request('https://test', { headers: { Authorization: `Bearer ${credential}` } })
+        )
+    ).toMatchObject({ ok: true, server: job.name, instanceId: instance.id });
+    expect(JSON.stringify(recipeForDeployment(job.deploymentId))).not.toContain(credential);
+    expect(
+        recipeForDeployment(job.deploymentId)!.recipe.files.some(
+            (file) => file.path === 'electric_cherry_bank.cfg'
+        )
+    ).toBe(false);
+});

tests/unit/runtime-config.test.ts

@@ -149,6 +149,7 @@ it.each([
             instance: backend.nativeInstance(a.server)!,
             serverKey: 'test-key',
             rconPassword: password,
+            bankToken: 'ec_bank_' + 'a'.repeat(43),
             rebuild: false
         });
         const actualConfig = prepared.recipe.files.find((file) => file.path === target)!;