From 54f43019942e19ca4e74ca92d8aed4d602fb4300 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 12 Oct 2026 22:20:45 -0400 Subject: [PATCH] Fix public bank endpoint rollout (Refs josh/tws-zombies-scripts#13) --- agent/CHANGELOG.md | 32 +++++++++++++ agent/ec_agent.py | 19 +++++--- agent/tests/test_agent.py | 82 ++++++++++++++++++++++++++++++++-- docs/guides/bank.md | 54 ++++++++++++++++++---- docs/guides/native-hosts.md | 11 ++++- static/agent/ec-agent.py | 19 +++++--- static/agent/manifest.json | 4 +- static/agent/manifest.json.sig | 8 ---- 8 files changed, 191 insertions(+), 38 deletions(-) create mode 100644 agent/CHANGELOG.md delete mode 100644 static/agent/manifest.json.sig diff --git a/agent/CHANGELOG.md b/agent/CHANGELOG.md new file mode 100644 index 0000000..dfd1a9a --- /dev/null +++ b/agent/CHANGELOG.md @@ -0,0 +1,32 @@ +# Agent changelog + +## 2026-10-12 + +### 1.9.1 — Public game-bank endpoint + +- Companion to [TWS scripts #13](https://gitea.thewrightserver.net/josh/tws-zombies-scripts/issues/13). + In `agent/ec_agent.py`, stop generating `tws_bank_url` in + `electric_cherry_bank.cfg`; continue injecting the private per-instance token. + Launch configuration still supplies `tws_server_id`. The host's enrolled + controller address and authentication remain unchanged. +- Recognize journals for `https://ec.thewrightserver.net/api/bank` and the legacy + bank URL derived from the enrolled controller. Reject other URLs and preserve + unresolved storage. Settlement still goes only through the enrolled controller's + authenticated, operation- and instance-scoped API. Original transaction IDs and + commit/cancel decisions are preserved across lost replies and retries. +- Compatibility: pair with T5/T6 bank scripts `2.0.0` and existing TWS plugins + `1.1.0+`. No new Python dependency, controller migration or protocol change. + Old scripts still fall back to the public endpoint when the dvar is absent. +- Deployment: package and offline-sign the release, update/restart host agents, + then rebuild through EC when a server's bank endpoint changes. Rebuild stops + the game and drains old decisions before starting the replacement or deleting + storage. An endpoint that already matches needs only a map restart for the + script update. Follow the [bank rollout procedure](../docs/guides/bank.md#upgrading-from-a-controller-derived-game-endpoint). + Packaging refreshes `static/agent/ec-agent.py` and `manifest.json` and removes + the old signature; a new offline signature is required before publishing. + No signing or deployment was performed. +- Regression coverage in `agent/tests/test_agent.py`: generated credentials, + public and legacy journals for both games, rebuild/removal, lost replies, + agent restart, unchanged identities and rejection of untrusted endpoints. + Live Windows/Plutonium endpoint migration and signed host rollout remain to + be validated. diff --git a/agent/ec_agent.py b/agent/ec_agent.py index be5dc3f..f83bae5 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -35,7 +35,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.9.0" +VERSION = "1.9.1" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -48,6 +48,7 @@ PLUTONIUM_MANIFEST_URL = "https://cdn.plutoniummod.com/updater/prod/info.json" PLUTONIUM_MANIFEST_LIMIT = 2 * 1024 * 1024 PLUTONIUM_FILE_LIMIT = 512 * 1024 * 1024 PLUTONIUM_CACHE_LIMIT = 4 * 1024 * 1024 * 1024 +BANK_GAME_ENDPOINT = "https://ec.thewrightserver.net/api/bank" ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$") AGENT_CONFIG = Path("/etc/electric-cherry/agent.json") AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py") @@ -1957,10 +1958,9 @@ class Agent: scoped(self.root / "agent" / "validation", path) if PurePosixPath(path).name != "electric_cherry_bank.cfg": raise Rejected("Invalid private bank config path") - controller = self.config["controller"].rstrip("/") - if any(char in controller for char in ('"', "\\", "\n", "\r", "\x00")): - raise Rejected("Invalid bank controller URL") - content = f'set tws_bank_token "{bank["token"]}"\nset tws_bank_url "{controller}/api/bank"\n'.encode() + # The game scripts use the public bank API; the host controller address + # is only for agent transport and must not become a game URL override. + content = f'set tws_bank_token "{bank["token"]}"\n'.encode() return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}] def write_initial_files(self, root, files): @@ -2057,8 +2057,13 @@ class Agent: try: journal = json.loads(raw) reports = journal["reports"] - if journal["endpoint"].rstrip("/") != self.config["controller"].rstrip("/") + "/api/bank": - raise AgentError("Bank journal belongs to a different controller; runtime storage retained") + # Accept the fixed game endpoint and this host's legacy generated + # endpoint so rebuild/removal can drain pre-upgrade decisions. + # Neither value controls transport: settlement still uses the + # enrolled controller, host credential and immutable operation. + endpoints = {BANK_GAME_ENDPOINT, self.config["controller"].rstrip("/") + "/api/bank"} + if journal["endpoint"].rstrip("/") not in endpoints: + raise AgentError("Bank journal has an unrecognized endpoint; runtime storage retained") if type(journal["version"]) is not int or journal["version"] != 1 or not isinstance(reports, list) or len(reports) > 4096: raise ValueError() # Controller independently binds this operation to the instance/server. diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 6dfce44..2e6c04b 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -1652,16 +1652,17 @@ class AgentTests(unittest.TestCase): operation = self.operation("instance.create", payload, "create") self.assertEqual(self.agent.execute(operation)["state"], "completed") path = self.agent.instance_dir(INSTANCE, 1) / "app/configs/electric_cherry_bank.cfg" - self.assertEqual(path.read_text(), f'set tws_bank_token "{secret}"\nset tws_bank_url "https://controller.example/api/bank"\n') + self.assertEqual(path.read_text(), f'set tws_bank_token "{secret}"\n') + self.assertEqual(self.agent.config["controller"], "https://controller.example") self.assertIn("bank.settlement", self.agent.inventory()["capabilities"]) payload["bank"]["token"] = secret + '\nset bad "1"' with self.assertRaises(AgentError): self.agent.initial_files(payload) - def bank_journal(self, decisions=("commit", "cancel"), game="t5"): - path = self.agent.instance_dir(INSTANCE, 1) / f"app/runtime/tws-bank/{game}/scope.json" + def bank_journal(self, decisions=("commit", "cancel"), game="t5", endpoint=module.BANK_GAME_ENDPOINT): + path = self.agent.instance_dir(INSTANCE, 1) / f"app/runtime/tws-bank/{game}/{module.digest(endpoint.encode())}.json" reports = [json.dumps({"server": "t5_zombie_test_1", "guid": "ace", "match": "match-one", "request_id": f"{game}-{i}", "operation": "deposit", "amount": 2000, "capacity": 1000000, "decision": decision}) for i, decision in enumerate(decisions)] - module.atomic_json(path, {"version": 1, "server": "t5_zombie_test_1", "endpoint": "https://controller.example/api/bank", "reports": reports}) + module.atomic_json(path, {"version": 1, "server": "t5_zombie_test_1", "endpoint": endpoint, "reports": reports}) return path def bank_acknowledgment(self, method, path, payload): @@ -1698,6 +1699,79 @@ class AgentTests(unittest.TestCase): self.assertFalse(path.exists()) self.assertEqual(self.docker.calls, calls) + def test_endpoint_upgrade_rebuild_retries_original_decisions_before_starting_public_bank(self): + secret = "ec_bank_" + "a" * 43 + payload = self.create() + payload["bank"] = {"token": secret, "configPath": "configs/electric_cherry_bank.cfg"} + paths = [self.bank_journal(game=game, endpoint=endpoint) + for game in ("t5", "t6") + for endpoint in (module.BANK_GAME_ENDPOINT, "https://controller.example/api/bank")] + originals = {path: path.read_bytes() for path in paths} + operation = self.operation("instance.rebuild", {"spec": payload, "nextGeneration": 2, "safetyOverride": True}, "rebuild") + applied, calls = {}, [] + + def acknowledge(method, path, data): + self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"]) + self.assertEqual(self.agent.config["controller"], "https://controller.example") + for journal, raw in originals.items(): + self.assertEqual(journal.read_bytes(), raw) + transfer = data["transfer"] + request_id = transfer["request_id"] + calls.append(request_id) + if request_id in applied: + self.assertEqual(applied[request_id], transfer) + applied[request_id] = transfer + response = self.bank_acknowledgment(method, path, data) + if len(calls) == 1: + raise AgentError("Lost reply after settlement") + return response + + with patch.object(self.agent, "request", side_effect=acknowledge): + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + self.agent.journal.close() + self.agent = Agent(self.config, self.docker) + with patch.object(self.agent, "request", side_effect=acknowledge): + self.assertEqual(self.agent.execute(operation)["state"], "failed") + retry = self.operation("instance.rebuild", operation["payload"], "replacement") + result = self.agent.execute(retry) + self.assertEqual(result["state"], "completed", result) + # Reconciliation drains once, then the replacement operation drains + # again before deleting the unchanged journals. All retries keep IDs. + self.assertEqual(len(calls), 17) + self.assertEqual(len(applied), 4) + self.assertEqual({item["decision"] for item in applied.values()}, {"commit", "cancel"}) + self.assertTrue(all(not path.exists() for path in paths)) + self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"]) + config = self.agent.instance_dir(INSTANCE, 2) / "app/configs/electric_cherry_bank.cfg" + self.assertEqual(config.read_text(), f'set tws_bank_token "{secret}"\n') + + def test_remove_drains_legacy_and_public_endpoints_without_following_journal_urls(self): + self.create() + paths = [self.bank_journal(game=game, endpoint=endpoint) + for game in ("t5", "t6") + for endpoint in (module.BANK_GAME_ENDPOINT, "https://controller.example/api/bank")] + with patch.object(self.agent, "request", side_effect=self.bank_acknowledgment) as request: + result = self.agent.execute(self.operation("instance.remove", {"safetyOverride": True}, "remove")) + self.assertEqual(result["state"], "completed", result) + self.assertEqual(request.call_count, 8) + self.assertTrue(all(not path.exists() for path in paths)) + + def test_unrecognized_bank_endpoints_retain_storage_without_settlement(self): + self.create() + path = self.bank_journal() + operation = self.operation("instance.remove", {"safetyOverride": True}, "remove") + for endpoint in ("https://untrusted.example/api/bank", "http://ec.thewrightserver.net/api/bank", + "https://ec.thewrightserver.net.evil/api/bank", module.BANK_GAME_ENDPOINT + "?redirect=evil", + "https://controller.example/other/api/bank", "", None): + with self.subTest(endpoint=endpoint): + journal = json.loads(path.read_text()) + journal["endpoint"] = endpoint + module.atomic_json(path, journal) + with patch.object(self.agent, "request") as request: + self.assertEqual(self.agent.execute(operation)["state"], "unknown") + request.assert_not_called() + self.assertTrue(path.exists()) + def test_rebuild_settles_before_starting_replacement_and_retains_unknown_decisions(self): payload = self.create() path = self.bank_journal(("commit",)) diff --git a/docs/guides/bank.md b/docs/guides/bank.md index 38dc6cc..98fd596 100644 --- a/docs/guides/bank.md +++ b/docs/guides/bank.md @@ -9,9 +9,10 @@ IDs. This release does not introduce separate bank-assisted rankings. ## Setup and rollout -1. Back up the controller database. Update every host agent to **1.9.0 or later** - before enabling the bank. This version settles game journals before removing - runtime storage; older agents do not. Package and offline-sign the agent +1. Back up the controller database. Update every host agent to **1.9.1 or later** + before installing the public-endpoint bank scripts. This version settles both + public and legacy controller-derived journals before removing runtime storage. + Package and offline-sign the agent release using the [host update workflow](native-hosts.md). 2. Deploy Electric Cherry and restart the controller. Migration 20 creates accounts, requests and the ledger; migration 21 adds instance credentials and @@ -21,13 +22,20 @@ IDs. This release does not introduce separate bank-assisted rankings. prepared deposits; withdrawals remain available. 4. Build/install TWS plugins `t5-plugin`/`t6-plugin` 1.1.0 or later and restart the **whole game-server process**. Install `tws_t5_bank.gsc` or - `tws_t6_bank.gsc` 1.0.0, common 1.0.0+ and telemetry 1.0.0+. + `tws_t6_bank.gsc` 2.0.0, common 1.0.0+ and telemetry 1.0.0+. + For existing servers changing endpoints, use the rebuild procedure below. 5. Create or rebuild the servers through EC. Launch preparation generates a cryptographically random `ec_bank_` credential for each immutable instance, stores it in the private controller database, and injects it through `configs/electric_cherry_bank.cfg`. The launch command executes this private - file and sets `tws_server_id`. The host agent writes the private file and derives `tws_bank_url` from its - enrolled controller URL. Use an HTTPS controller URL. + file and sets `tws_server_id`. The host agent writes only `tws_bank_token` in + that private file; it no longer generates `tws_bank_url`. + +Both game scripts always send bank requests to +`https://ec.thewrightserver.net/api/bank`, even if an old `tws_bank_url` remains +in configuration. The host agent's enrolled HTTPS controller URL is separate +and continues to carry agent traffic, including authenticated bank settlement. +It may differ from the public endpoint reachable by game containers. There is **no `EC_BANK_TOKENS` environment map** and no per-server secret setup. The private configuration is sent only in that host's authenticated operation @@ -49,6 +57,28 @@ transfers while the old controller/configuration still runs, then deploy and rebuild. Remove the obsolete environment map and any manually supplied token. Do not replace identities or discard pending journals to force an upgrade. +### Upgrading from a controller-derived game endpoint + +Update agents to `1.9.1+` first. If an existing server's `tws_bank_url` differs +from `https://ec.thewrightserver.net/api/bank`, rebuild that server through EC +with bank scripts `2.0.0`. The agent stops the old game, settles its pending +commit/cancel decisions with their original request IDs, then starts the +replacement and removes old storage. The server identity and instance credential +are preserved. Lost acknowledgments can replay the same decisions; EC's retained +request IDs prevent duplicate ledger changes. An outage or invalid journal +retains storage and blocks replacement until recovery succeeds. + +Do not deploy the new scripts with only a map restart when the endpoint changes: +native journals are keyed by server and endpoint, and the new scripts cannot load +the old scope. Do not rename, rewrite or delete pending journals. Keep the +agent's enrolled controller URL unchanged until legacy journals drain; the +agent accepts only the fixed public endpoint and the legacy `/api/bank` URL +derived from that enrolled controller. Other legacy URLs require recovery with +their original configuration before switching. If the old URL already matches +the exact public endpoint, existing journals retain their identity and a map +restart is sufficient for the script upgrade. No new native plugin build is +required beyond `1.1.0`. + ## Machine API All routes below are `POST /api/bank/`, with `Content-Type: @@ -136,14 +166,15 @@ original journal to resolve holds; never delete a pending journal to unblock the bank. If storage is permanently lost, reconcile against the retained request and ledger records before any manual repair. -During removal, rebuild, host migration or launch cancellation, agent 1.9.0+ +During removal, rebuild, host migration or launch cancellation, agent 1.9.1+ stops the game and drains its journals **before** deleting runtime files or starting a replacement. It uses its host credential at `POST /api/agent/bank/settle`; EC checks the assigned host, immutable instance, unfinished teardown operation, server and original request identity. This endpoint permits only commit/cancel, never new deposits or withdrawals. Journal -URLs must match that host's configured controller; the agent never follows a -journal URL. Corruption, a mismatched controller, failed acknowledgment or an +URLs must match the public game-bank endpoint or that host's legacy +controller-derived bank endpoint; the agent never follows a journal URL. +Corruption, an unrecognized endpoint, failed acknowledgment or an outage retains storage and leaves the operation unresolved for retry. An interrupted rebuild uses the existing rebuild recovery workflow to retry with a fresh operation after settlement. Do not manually erase its old directory. @@ -170,3 +201,8 @@ telemetry, `.help`, score HUD updates, death/respawn/disconnect during requests, lost prepare/commit replies, controller outage, disk failures and map/process restart with pending work. Confirm earned score and leaderboard behavior, and verify that another server's credential and the telemetry credential fail. +For the endpoint upgrade, also test a different enrolled controller address and +a stale `tws_bank_url`, `.b` with an empty balance, `.w all` with no funds, and +rebuild/removal with both legacy and public journals. Drop a settlement reply, +restart the agent and verify the original IDs settle without duplicate ledger +changes before the replacement starts or runtime storage is removed. diff --git a/docs/guides/native-hosts.md b/docs/guides/native-hosts.md index f8758b5..702a0ad 100644 --- a/docs/guides/native-hosts.md +++ b/docs/guides/native-hosts.md @@ -1038,9 +1038,18 @@ game hosts. Run: ### Bank journals during server teardown -Agent 1.9.0 adds `bank.settlement`: removal, rebuild, migration and creation +Agent 1.9.0 introduced `bank.settlement`: removal, rebuild, migration and creation cancellation finish pending TWS bank decisions before deleting the old runtime. An unavailable controller or corrupt journal leaves the operation unresolved and preserves storage. Update all agents before enabling the [player bank](bank.md). EC generates and injects bank credentials per instance; no token map or manual server-secret provisioning is required. + +Agent **1.9.1+** recognizes both the fixed public game-bank endpoint +`https://ec.thewrightserver.net/api/bank` and its own legacy controller-derived +bank endpoint in journals. It stops generating `tws_bank_url` in the private +game configuration. Agent transport still uses the enrolled controller URL and +host credential; journal URLs never determine request destinations. Upgrade +agents first, then [rebuild servers changing bank endpoints](bank.md#upgrading-from-a-controller-derived-game-endpoint) +to drain pending decisions before the new scripts start. See the +[agent changelog](../../agent/CHANGELOG.md) for release and validation details. diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py index be5dc3f..f83bae5 100644 --- a/static/agent/ec-agent.py +++ b/static/agent/ec-agent.py @@ -35,7 +35,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.9.0" +VERSION = "1.9.1" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 MANIFEST_MAX_ENTRIES = 10000 @@ -48,6 +48,7 @@ PLUTONIUM_MANIFEST_URL = "https://cdn.plutoniummod.com/updater/prod/info.json" PLUTONIUM_MANIFEST_LIMIT = 2 * 1024 * 1024 PLUTONIUM_FILE_LIMIT = 512 * 1024 * 1024 PLUTONIUM_CACHE_LIMIT = 4 * 1024 * 1024 * 1024 +BANK_GAME_ENDPOINT = "https://ec.thewrightserver.net/api/bank" ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$") AGENT_CONFIG = Path("/etc/electric-cherry/agent.json") AGENT_EXECUTABLE = Path("/usr/local/libexec/electric-cherry-agent.py") @@ -1957,10 +1958,9 @@ class Agent: scoped(self.root / "agent" / "validation", path) if PurePosixPath(path).name != "electric_cherry_bank.cfg": raise Rejected("Invalid private bank config path") - controller = self.config["controller"].rstrip("/") - if any(char in controller for char in ('"', "\\", "\n", "\r", "\x00")): - raise Rejected("Invalid bank controller URL") - content = f'set tws_bank_token "{bank["token"]}"\nset tws_bank_url "{controller}/api/bank"\n'.encode() + # The game scripts use the public bank API; the host controller address + # is only for agent transport and must not become a game URL override. + content = f'set tws_bank_token "{bank["token"]}"\n'.encode() return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}] def write_initial_files(self, root, files): @@ -2057,8 +2057,13 @@ class Agent: try: journal = json.loads(raw) reports = journal["reports"] - if journal["endpoint"].rstrip("/") != self.config["controller"].rstrip("/") + "/api/bank": - raise AgentError("Bank journal belongs to a different controller; runtime storage retained") + # Accept the fixed game endpoint and this host's legacy generated + # endpoint so rebuild/removal can drain pre-upgrade decisions. + # Neither value controls transport: settlement still uses the + # enrolled controller, host credential and immutable operation. + endpoints = {BANK_GAME_ENDPOINT, self.config["controller"].rstrip("/") + "/api/bank"} + if journal["endpoint"].rstrip("/") not in endpoints: + raise AgentError("Bank journal has an unrecognized endpoint; runtime storage retained") if type(journal["version"]) is not int or journal["version"] != 1 or not isinstance(reports, list) or len(reports) > 4096: raise ValueError() # Controller independently binds this operation to the instance/server. diff --git a/static/agent/manifest.json b/static/agent/manifest.json index 4688fcf..638c56e 100644 --- a/static/agent/manifest.json +++ b/static/agent/manifest.json @@ -1,8 +1,8 @@ { - "version": "1.9.0", + "version": "1.9.1", "protocolVersion": 1, "platform": "debian-amd64", - "sha256": "07f91ba8780c1e2b0a9c0ab3534c1731d4246d3f5d84fece14691c930dcb5e90", + "sha256": "c3c9fc251c93802e8016b67dc3fba409df00de6681751b9ba231b80d702c7333", "updaterProtocolVersion": 2, "updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb", "installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231" diff --git a/static/agent/manifest.json.sig b/static/agent/manifest.json.sig deleted file mode 100644 index 7c71eed..0000000 --- a/static/agent/manifest.json.sig +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN PGP SIGNATURE----- - -iJEEABYKADkWIQSpocb6VBfQzvdIh9HF6fdSNniN+gUCas2ANxsUgAAAAAAEAA5t -YW51MiwyLjUrMS4xMiwwLDMACgkQxen3UjZ4jfotJAD/df3qwKWk45tYFghvCJZm -viIxpqvNvCpxkvjgicV+d00BAIJRbVswzE46KRMimu8P61re0b2ppqEEfRmQc2N6 -3YkE -=cojL ------END PGP SIGNATURE-----