Compare commits

..
3 Commits
Author SHA1 Message Date
josh 0a98ea319c Sign packaged agent release
CI / checks (push) Successful in 5m30s
2026-10-12 20:50:26 -04:00
josh d14c6a7ff4 Provision instance bank credentials automatically (Refs #146) 2026-10-12 20:44:01 -04:00
josh e071e5b574 Add persistent cross-game player bank (Refs #146) 2026-10-12 20:21:58 -04:00
27 changed files with 1419 additions and 21 deletions
+2
View File
@@ -7,6 +7,8 @@ EC_SETUP_TOKEN=
# Game reporters send Authorization: Bearer <token> or X-EC-Telemetry-Token. # Game reporters send Authorization: Bearer <token> or X-EC-Telemetry-Token.
# Telemetry ingestion is disabled until a token of at least 32 chars is set. # Telemetry ingestion is disabled until a token of at least 32 chars is set.
EC_TELEMETRY_TOKEN= EC_TELEMETRY_TOKEN=
# Bank credentials are generated and injected automatically per native instance.
EC_BANK_MAX_BALANCE=250000
# Persistent SQLite metadata, game-file storage, and beta archives. # Persistent SQLite metadata, game-file storage, and beta archives.
# For an existing deployment, retain its original data directory. # For an existing deployment, retain its original data directory.
EC_DATA_DIR=/root/zombies/.electric-cherry EC_DATA_DIR=/root/zombies/.electric-cherry
+96 -6
View File
@@ -35,7 +35,7 @@ import urllib.parse
import urllib.request import urllib.request
import uuid import uuid
VERSION = "1.8.7" VERSION = "1.9.0"
LABEL = "io.electric-cherry" LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024 MAX_FILE = 32 * 1024 * 1024
MANIFEST_MAX_ENTRIES = 10000 MANIFEST_MAX_ENTRIES = 10000
@@ -1503,7 +1503,7 @@ class Agent:
if line.startswith("PRETTY_NAME="): if line.startswith("PRETTY_NAME="):
os_name = line.split("=", 1)[1].strip('"') os_name = line.split("=", 1)[1].strip('"')
update = self.update_inventory() update = self.update_inventory()
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
plutonium_cache = self.plutonium_cache_inventory() plutonium_cache = self.plutonium_cache_inventory()
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
@@ -1657,7 +1657,7 @@ class Agent:
# record. Whenever preparation was recorded, enforce that exact image. # record. Whenever preparation was recorded, enforce that exact image.
if image in prepared and prepared[image] != image_id: if image in prepared and prepared[image] != image_id:
raise AgentError("Prepared runtime image has changed. An admin must prepare it again before launching.") raise AgentError("Prepared runtime image has changed. An admin must prepare it again before launching.")
files = payload.get("files", []) files = self.initial_files(payload)
if not isinstance(files, list) or len(files) > 10000: if not isinstance(files, list) or len(files) > 10000:
raise Rejected("Invalid initial file list") raise Rejected("Invalid initial file list")
total, paths = 0, set() total, paths = 0, set()
@@ -1911,7 +1911,7 @@ class Agent:
self.operation_phase = "seeding-runtime" self.operation_phase = "seeding-runtime"
self.seed_plutonium(app, image) self.seed_plutonium(app, image)
self.operation_phase = "writing-create-files" self.operation_phase = "writing-create-files"
self.write_initial_files(app, payload.get("files", [])) self.write_initial_files(app, self.initial_files(payload))
self.operation_phase = "persisting-create-metadata" self.operation_phase = "persisting-create-metadata"
atomic_json(directory / "spec.json", payload) atomic_json(directory / "spec.json", payload)
env_path = directory / "environment" env_path = directory / "environment"
@@ -1944,6 +1944,25 @@ class Agent:
if self.executing_id and self.executing_id in self.cancelled_creations: if self.executing_id and self.executing_id in self.cancelled_creations:
raise AgentError("Creation cancelled; waiting for instance cleanup") raise AgentError("Creation cancelled; waiting for instance cleanup")
def initial_files(self, payload):
files = payload.get("files", [])
if not isinstance(files, list):
raise Rejected("Invalid initial file list")
bank = payload.get("bank")
if bank is None:
return files
if not isinstance(bank, dict) or not isinstance(bank.get("token"), str) or not re.fullmatch(r"ec_bank_[A-Za-z0-9_-]{32,128}", bank["token"]):
raise Rejected("Invalid instance bank credential")
path = bank.get("configPath")
scoped(self.root / "agent" / "validation", path)
if PurePosixPath(path).name != "electric_cherry_bank.cfg":
raise Rejected("Invalid private bank config path")
controller = self.config["controller"].rstrip("/")
if any(char in controller for char in ('"', "\\", "\n", "\r", "\x00")):
raise Rejected("Invalid bank controller URL")
content = f'set tws_bank_token "{bank["token"]}"\nset tws_bank_url "{controller}/api/bank"\n'.encode()
return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}]
def write_initial_files(self, root, files): def write_initial_files(self, root, files):
if not isinstance(files, list) or len(files) > 10000: if not isinstance(files, list) or len(files) > 10000:
raise AgentError("Invalid file list") raise AgentError("Invalid file list")
@@ -1992,6 +2011,72 @@ class Agent:
if (time.monotonic() - started) * 1000 >= ttl: if (time.monotonic() - started) * 1000 >= ttl:
raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation") raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation")
def settle_bank(self, instance_id, generation, operation_id=None):
"""Drain stopped runtime journals before discarding any generation's storage.
Use only our configured controller and host credential. Never follow an
endpoint or symlink supplied by a game journal. Lost replies are retried
with the original request identity; files remain until teardown succeeds.
"""
app = scoped(self.instance_dir(instance_id, generation), "app")
if not app.exists():
return
if self.inspect(instance_id, generation)["running"]:
raise AgentError("Bank settlement requires a stopped instance")
operation_id = operation_id or self.executing_id
visited = 0
deadline = time.monotonic() + 30
journals = []
def unreadable(_error):
raise AgentError("Bank journal discovery failed; runtime storage retained")
for directory, directories, files in os.walk(app, followlinks=False, onerror=unreadable):
relative = Path(directory).relative_to(app)
inside_bank = "tws-bank" in relative.parts
visited += 1
if visited > 100000 or time.monotonic() > deadline:
raise AgentError("Bank journal discovery limit exceeded; runtime storage retained")
for name in directories + files:
if (inside_bank or name == "tws-bank") and (Path(directory) / name).is_symlink():
raise AgentError("Bank journal paths must not be symlinks")
if not inside_bank:
continue
for name in files:
if name.endswith(".json"):
journals.append((relative / name).as_posix())
if len(journals) > 4096:
raise AgentError("Too many bank journals; runtime storage retained")
for path in sorted(journals):
try:
raw = managed_read(app, path)
except Rejected:
raise AgentError("Bank journal cannot be read; runtime storage retained") from None
if raw is None or len(raw) > 16 * 1024 * 1024:
raise AgentError("Bank journal is unavailable or oversized")
try:
journal = json.loads(raw)
reports = journal["reports"]
if journal["endpoint"].rstrip("/") != self.config["controller"].rstrip("/") + "/api/bank":
raise AgentError("Bank journal belongs to a different controller; runtime storage retained")
if type(journal["version"]) is not int or journal["version"] != 1 or not isinstance(reports, list) or len(reports) > 4096:
raise ValueError()
# Controller independently binds this operation to the instance/server.
for report in reports:
if not isinstance(report, str) or len(report.encode()) > 20000:
raise ValueError()
transfer = json.loads(report)
action = transfer["decision"]
if action not in ("commit", "cancel") or transfer["server"] != journal["server"] or not operation_id:
raise ValueError()
response = self.request("POST", "/api/agent/bank/settle", {
"operationId": operation_id, "action": action, "transfer": transfer})
expected = ("committed",) if action == "commit" else ("cancelled", "rejected")
if not isinstance(response, dict) or response.get("request_id") != transfer["request_id"] or response.get("state") not in expected:
raise AgentError("Bank settlement was not acknowledged; runtime storage retained")
except (ValueError, TypeError, KeyError, AttributeError):
raise AgentError("Bank journal is invalid; runtime storage retained") from None
def lifecycle(self, kind, instance_id, generation, payload): def lifecycle(self, kind, instance_id, generation, payload):
unhealthy_recovery = payload.get("unhealthyRecovery") is True unhealthy_recovery = payload.get("unhealthyRecovery") is True
if unhealthy_recovery and (kind not in ("instance.restart", "instance.rebuild") or payload.get("stageId") or "autoscaling" in payload): if unhealthy_recovery and (kind not in ("instance.restart", "instance.rebuild") or payload.get("stageId") or "autoscaling" in payload):
@@ -2043,6 +2128,7 @@ class Agent:
if health["exists"]: if health["exists"]:
self.docker.call("container", "stop", "--time", "30", name) self.docker.call("container", "stop", "--time", "30", name)
self.docker.call("container", "rm", name) self.docker.call("container", "rm", name)
self.settle_bank(instance_id, generation)
if kind == "instance.rebuild": if kind == "instance.rebuild":
result = self.create(instance_id, next_generation, spec) result = self.create(instance_id, next_generation, spec)
self.operation_phase = "removing-old-instance" self.operation_phase = "removing-old-instance"
@@ -2062,7 +2148,7 @@ class Agent:
raise AgentError("Unsupported instance operation") raise AgentError("Unsupported instance operation")
return {**self.inspect(instance_id, generation), "before": before} return {**self.inspect(instance_id, generation), "before": before}
def cancel_creation(self, instance_id, generation, payload): def cancel_creation(self, instance_id, generation, payload, operation_id=None):
"""Explicit launch cancellation, fenced to its immutable creation identity. """Explicit launch cancellation, fenced to its immutable creation identity.
The operation loop serializes this with create. Once delivered, removal The operation loop serializes this with create. Once delivered, removal
@@ -2106,6 +2192,7 @@ class Agent:
self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, target_generation)) self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, target_generation))
if self.inspect(instance_id, target_generation)["exists"]: if self.inspect(instance_id, target_generation)["exists"]:
raise AgentError("Cancelled creation container still exists") raise AgentError("Cancelled creation container still exists")
self.settle_bank(instance_id, target_generation, operation_id)
directory = self.instance_dir(instance_id, target_generation) directory = self.instance_dir(instance_id, target_generation)
if directory.exists(): if directory.exists():
shutil.rmtree(directory) shutil.rmtree(directory)
@@ -2747,6 +2834,7 @@ class Agent:
if not unchanged_generation() or not both_missing(): if not unchanged_generation() or not both_missing():
return row return row
self.settle_bank(instance_id, generation, operation["id"])
# Preparation can leave runtime/configuration files before create's # Preparation can leave runtime/configuration files before create's
# checkpoint. A fresh operation must not inherit those partial files. # checkpoint. A fresh operation must not inherit those partial files.
expected_spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")} expected_spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
@@ -2793,7 +2881,7 @@ class Agent:
if kind == "instance.remove" and "autoscaling" in payload and not checkpoint: if kind == "instance.remove" and "autoscaling" in payload and not checkpoint:
return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision") return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision")
if kind == "instance.cancel-create": if kind == "instance.cancel-create":
return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload)) return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload, operation["id"]))
if kind == "instance.create": if kind == "instance.create":
marker = self.instance_dir(instance_id, generation).parent / "current.json" marker = self.instance_dir(instance_id, generation).parent / "current.json"
if marker.exists(): if marker.exists():
@@ -2885,6 +2973,7 @@ class Agent:
directory = self.instance_dir(instance_id, observed_generation) directory = self.instance_dir(instance_id, observed_generation)
atomic_json(directory.parent / "current.json", {"generation": observed_generation}) atomic_json(directory.parent / "current.json", {"generation": observed_generation})
if kind == "instance.rebuild" and not self.inspect(instance_id, generation)["exists"]: if kind == "instance.rebuild" and not self.inspect(instance_id, generation)["exists"]:
self.settle_bank(instance_id, generation, operation["id"])
old = self.instance_dir(instance_id, generation) old = self.instance_dir(instance_id, generation)
if old.exists(): if old.exists():
shutil.rmtree(old) shutil.rmtree(old)
@@ -2899,6 +2988,7 @@ class Agent:
elif kind == "instance.remove": elif kind == "instance.remove":
proven = not health["exists"] proven = not health["exists"]
if proven: if proven:
self.settle_bank(instance_id, generation, operation["id"])
directory = self.instance_dir(instance_id, generation) directory = self.instance_dir(instance_id, generation)
if directory.exists(): if directory.exists():
shutil.rmtree(directory) shutil.rmtree(directory)
+103
View File
@@ -1644,6 +1644,109 @@ class AgentTests(unittest.TestCase):
self.assertEqual(self.agent.execute(self.operation("instance.remove"))["state"], "failed") self.assertEqual(self.agent.execute(self.operation("instance.remove"))["state"], "failed")
self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls)) self.assertFalse(any(call[:2] == ("container", "stop") for call in self.docker.calls))
def test_create_materializes_private_bank_config_from_instance_credential(self):
secret = "ec_bank_" + "a" * 43
payload = {"name": "server", "game": "t5", "libraryId": self.library_id,
"image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960,
"bank": {"token": secret, "configPath": "configs/electric_cherry_bank.cfg"}}
operation = self.operation("instance.create", payload, "create")
self.assertEqual(self.agent.execute(operation)["state"], "completed")
path = self.agent.instance_dir(INSTANCE, 1) / "app/configs/electric_cherry_bank.cfg"
self.assertEqual(path.read_text(), f'set tws_bank_token "{secret}"\nset tws_bank_url "https://controller.example/api/bank"\n')
self.assertIn("bank.settlement", self.agent.inventory()["capabilities"])
payload["bank"]["token"] = secret + '\nset bad "1"'
with self.assertRaises(AgentError):
self.agent.initial_files(payload)
def bank_journal(self, decisions=("commit", "cancel"), game="t5"):
path = self.agent.instance_dir(INSTANCE, 1) / f"app/runtime/tws-bank/{game}/scope.json"
reports = [json.dumps({"server": "t5_zombie_test_1", "guid": "ace", "match": "match-one", "request_id": f"{game}-{i}", "operation": "deposit", "amount": 2000, "capacity": 1000000, "decision": decision}) for i, decision in enumerate(decisions)]
module.atomic_json(path, {"version": 1, "server": "t5_zombie_test_1", "endpoint": "https://controller.example/api/bank", "reports": reports})
return path
def bank_acknowledgment(self, method, path, payload):
self.assertEqual((method, path), ("POST", "/api/agent/bank/settle"))
self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"])
return {"state": "committed" if payload["action"] == "commit" else "cancelled", "request_id": payload["transfer"]["request_id"]}
def test_remove_settles_both_game_journals_after_stop_before_deleting_storage(self):
self.create()
paths = [self.bank_journal(game=game) for game in ("t5", "t6")]
def acknowledge(*args):
self.assertTrue(all(path.exists() for path in paths))
self.assertEqual(args[2]["operationId"], "remove")
return self.bank_acknowledgment(*args)
with patch.object(self.agent, "request", side_effect=acknowledge) as request:
result = self.agent.execute(self.operation("instance.remove", {"safetyOverride": True}, "remove"))
self.assertEqual(result["state"], "completed", result)
self.assertEqual(request.call_count, 4)
self.assertTrue(all(not path.exists() for path in paths))
def test_remove_retains_bank_journal_during_outage_and_resumes_after_agent_restart(self):
self.create()
path = self.bank_journal()
operation = self.operation("instance.remove", {"safetyOverride": True}, "remove")
with patch.object(self.agent, "request", side_effect=AgentError("Controller request failed")):
self.assertEqual(self.agent.execute(operation)["state"], "unknown")
self.assertEqual(self.agent.execute(operation)["state"], "unknown")
self.assertTrue(path.exists())
self.agent.journal.close()
self.agent = Agent(self.config, self.docker)
calls = list(self.docker.calls)
with patch.object(self.agent, "request", side_effect=self.bank_acknowledgment):
self.assertEqual(self.agent.execute(operation)["state"], "completed")
self.assertFalse(path.exists())
self.assertEqual(self.docker.calls, calls)
def test_rebuild_settles_before_starting_replacement_and_retains_unknown_decisions(self):
payload = self.create()
path = self.bank_journal(("commit",))
operation = self.operation("instance.rebuild", {"spec": payload, "nextGeneration": 2, "safetyOverride": True}, "rebuild")
with patch.object(self.agent, "request", return_value={"state": "committed", "request_id": "wrong"}):
self.assertEqual(self.agent.execute(operation)["state"], "unknown")
self.assertTrue(path.exists())
self.assertFalse(self.agent.inspect(INSTANCE, 2)["exists"])
with patch.object(self.agent, "request", side_effect=self.bank_acknowledgment):
# Existing rebuild recovery records the interrupted create as failed;
# a fresh operation can then rebuild without replaying the old command.
self.assertEqual(self.agent.execute(operation)["state"], "failed")
retry = self.operation("instance.rebuild", operation["payload"], "replacement")
self.assertEqual(self.agent.execute(retry)["state"], "completed")
self.assertFalse(path.exists())
self.assertTrue(self.agent.inspect(INSTANCE, 2)["running"])
def test_cancel_creation_retries_bank_settlement_before_deleting_runtime(self):
self.create()
path = self.bank_journal(("cancel",))
operation = self.operation("instance.cancel-create", {"createOperationId": "create"}, "cancel")
with patch.object(self.agent, "request", side_effect=AgentError("offline")):
self.assertEqual(self.agent.execute(operation)["state"], "unknown")
self.assertTrue(path.exists())
with patch.object(self.agent, "request", side_effect=self.bank_acknowledgment):
self.assertEqual(self.agent.execute(operation)["state"], "completed")
self.assertFalse(path.exists())
def test_malformed_or_symlinked_bank_journal_blocks_storage_deletion(self):
self.create()
path = self.bank_journal()
path.write_text('{broken')
operation = self.operation("instance.remove", {"safetyOverride": True}, "remove")
with patch.object(self.agent, "request") as request:
self.assertEqual(self.agent.execute(operation)["state"], "unknown")
self.assertTrue(path.exists())
outside = Path(self.temp.name) / "outside.json"
outside.write_text('{"reports": []}')
self.bank_journal()
journal = json.loads(path.read_text())
journal["endpoint"] = "https://untrusted.example/api/bank"
path.write_text(json.dumps(journal))
self.assertEqual(self.agent.execute(operation)["state"], "unknown")
path.unlink()
path.symlink_to(outside)
self.assertEqual(self.agent.execute(operation)["state"], "unknown")
self.assertTrue(path.is_symlink())
request.assert_not_called()
def test_rebuild_moves_generation_and_rejects_old_commands(self): def test_rebuild_moves_generation_and_rejects_old_commands(self):
payload = self.create() payload = self.create()
self.agent.rcon = lambda *_: {"ok": True, "players": 0} self.agent.rcon = lambda *_: {"ok": True, "players": 0}
+1
View File
@@ -15,6 +15,7 @@ services:
ORIGIN: ${EC_PUBLIC_URL:-http://localhost:3000} ORIGIN: ${EC_PUBLIC_URL:-http://localhost:3000}
EC_SETUP_TOKEN: ${EC_SETUP_TOKEN:-} EC_SETUP_TOKEN: ${EC_SETUP_TOKEN:-}
EC_TELEMETRY_TOKEN: ${EC_TELEMETRY_TOKEN:-} EC_TELEMETRY_TOKEN: ${EC_TELEMETRY_TOKEN:-}
EC_BANK_MAX_BALANCE: ${EC_BANK_MAX_BALANCE:-250000}
EC_AUTH_ALLOW_HTTP: ${EC_AUTH_ALLOW_HTTP:-false} EC_AUTH_ALLOW_HTTP: ${EC_AUTH_ALLOW_HTTP:-false}
# Legacy host imports stream whole game files; buffering routes enforce their own caps. # Legacy host imports stream whole game files; buffering routes enforce their own caps.
BODY_SIZE_LIMIT: 'Infinity' BODY_SIZE_LIMIT: 'Infinity'
+1
View File
@@ -25,6 +25,7 @@ and launch your first server.
- [Beta test plans](guides/beta-test-plans.md) — game reporters, test requirements, coverage, and outcomes. - [Beta test plans](guides/beta-test-plans.md) — game reporters, test requirements, coverage, and outcomes.
- [Plugin releases](guides/plugin-releases.md) — GitHub release discovery and imports. - [Plugin releases](guides/plugin-releases.md) — GitHub release discovery and imports.
- [Metrics](guides/metrics.md) — utilization, capacity pressure, and hosting experiments. - [Metrics](guides/metrics.md) — utilization, capacity pressure, and hosting experiments.
- [Player bank](guides/bank.md) — shared balances, per-server credentials, transfers and recovery.
- [Leaderboard and matches](guides/leaderboard.md) — player statistics, match history, and the read API. - [Leaderboard and matches](guides/leaderboard.md) — player statistics, match history, and the read API.
## Development ## Development
+172
View File
@@ -0,0 +1,172 @@
# Player bank
The bank stores one points balance per exact GUID across T5 and T6. Players use
`.b`, `.d <amount|all>` and `.w <amount|all>` with the TWS bank scripts. Deposits
move spendable match points into the bank; withdrawals return bank points to
the current match. Neither changes earned `score_total`. Bank-assisted games
still count toward the existing leaderboards; the ledger records their match
IDs. This release does not introduce separate bank-assisted rankings.
## Setup and rollout
1. Back up the controller database. Update every host agent to **1.9.0 or later**
before enabling the bank. This version settles game journals before removing
runtime storage; older agents do not. Package and offline-sign the agent
release using the [host update workflow](native-hosts.md).
2. Deploy Electric Cherry and restart the controller. Migration 20 creates
accounts, requests and the ledger; migration 21 adds instance credentials and
binds existing requests to their currently active native instance, if any.
3. Set `EC_BANK_MAX_BALANCE` (default 250000, range 1–1000000). Invalid configuration
refuses new deposits. Lowering the limit preserves existing balances and
prepared deposits; withdrawals remain available.
4. Build/install TWS plugins `t5-plugin`/`t6-plugin` 1.1.0 or later and restart
the **whole game-server process**. Install `tws_t5_bank.gsc` or
`tws_t6_bank.gsc` 1.0.0, common 1.0.0+ and telemetry 1.0.0+.
5. Create or rebuild the servers through EC. Launch preparation generates a
cryptographically random `ec_bank_` credential for each immutable instance,
stores it in the private controller database, and injects it through
`configs/electric_cherry_bank.cfg`. The launch command executes this private
file and sets `tws_server_id`. The host agent writes the private file and derives `tws_bank_url` from its
enrolled controller URL. Use an HTTPS controller URL.
There is **no `EC_BANK_TOKENS` environment map** and no per-server secret setup.
The private configuration is sent only in that host's authenticated operation
payload. It is excluded from shared recipes, promoted files and blob downloads.
Do not upload `electric_cherry_bank.cfg`: its filename is reserved. Protect the
controller database, host operation journal and per-instance runtime directory,
which contain private credentials.
Retries, rebuilds and host moves of the same instance reuse its credential.
Retiring the instance immediately revokes authentication. Creating another
server with the same display name produces a new instance ID and secret; it
cannot finish the old instance's requests. Accounts remain attached to player
GUIDs, so balances survive the servers. Telemetry tokens and browser sessions
cannot authorize bank requests. Host credentials can only settle decisions
through the separate, operation-scoped teardown endpoint.
For an upgrade from the initial static-token implementation, drain outstanding
transfers while the old controller/configuration still runs, then deploy and
rebuild. Remove the obsolete environment map and any manually supplied token.
Do not replace identities or discard pending journals to force an upgrade.
## Machine API
All routes below are `POST /api/bank/<action>`, with `Content-Type:
application/json` and `Authorization: Bearer <server-bank-token>`. Responses
are never cached. Bodies are capped at 4096 bytes. GUIDs and request IDs stay
strings; never convert them to numbers.
| Action | Body | Meaning |
| ---------- | ------------------------- | -------------------------------------------------------------------- |
| `balance` | `server`, `guid` | Read the shared balance and currently available points |
| `deposit` | Transfer fields below | Reserve deposit capacity without crediting the account yet |
| `withdraw` | Transfer fields below | Reserve bank funds without debiting the account yet |
| `commit` | Identical transfer fields | Apply a prepared transfer once and append its ledger entry |
| `cancel` | Identical transfer fields | Release a reservation; also prevents a delayed prepare from applying |
Transfer example:
```json
{
"server": "t5_zombie_theater_1",
"guid": "18446744073709551615",
"match": "reporter-match-key",
"request_id": "unique-process-token-17",
"operation": "deposit",
"amount": 2000,
"capacity": 995000
}
```
`match` is the reporter's telemetry match key, **not** Electric Cherry's numeric
match ID. `request_id` is unique for the server and at most 64 characters.
`amount` is a positive integer up to 1000000; withdrawal also accepts `"all"`.
The script resolves deposit `all` to its captured spendable points. `capacity`
is the current room below the game's bank withdrawal cap of 1000000 points.
An `all` withdrawal that exceeds this room is refused, rather than truncated.
Preparation requires the GUID to be present in the server's latest active match,
with the exact telemetry match key. Late joiners may need to wait for the next
30-second heartbeat. Unknown/destroyed servers, absent players and old matches
cannot prepare transfers. Settlement of an existing decision remains possible
after match completion or retention. Host teardown can finish saved decisions
after retirement; the retired game credential itself no longer authenticates.
Successful application responses include `ok`, `state`, `code`, `request_id`,
`amount` and `balance`. States are `balance`, `prepared`, `committed`, `cancelled`
or `rejected`. For balance reads, `amount` is the available balance. A business
rejection uses HTTP 200 with `ok: false`, `state: "rejected"` and a code such as
`insufficient_balance`, `empty`, `bank_limit`, `points_limit`, `not_in_match` or
`pending`. It never changes points. Authentication failures use 401/403,
malformed input 400/413/415, identity conflicts 409, and unavailable service 503.
Repeated requests return the saved result/state without applying a second
change. Settled IDs never reopen; a committed result retains its original
resulting balance even if later transfers change the account. Reusing an ID
with different transfer fields returns 409. A GUID has at most one prepared
transfer across all servers. SQLite transactions serialize checks, account
updates and ledger insertion, so concurrent requests cannot overspend funds or
deposit past the configured maximum.
## Lost responses and recovery
The game saves a cancellation decision **before** sending a prepare request.
Only a validated prepared reply lets it recheck the player's life, current
spendable points and withdrawal room. It then records a commit decision and
updates spendable points without yielding. The sender retries settlement with
the original identity, at delays of 1, 2, 4, 8, 16, then 30 seconds. Invalid,
truncated or uncorrelated replies never authorize points changes.
A failed/uncertain prepare leaves game points unchanged and queues cancellation.
Cancellation can arrive first: its tombstone blocks the delayed prepare. Death,
respawn, disconnect or match end before application also cancels. After a commit
decision, retries only finish the ledger; they never change game points again.
These points now belong to the current match, so an applied withdrawal is not
automatically refunded if that match ends or its server crashes.
Keep `tws-bank/t5/` or `tws-bank/t6/`, relative to the game process working
directory, on persistent writable storage with a
single process per server/endpoint. Journals contain transfer data, not secrets.
On a map/process restart, saved decisions settle before new bank commands.
Unreadable storage disables the bank; failed writes prevent points changes.
Pending decisions remain until acknowledged, even on credential errors. Bank
holds do not expire automatically, because expiring a saved commit can lose a
deposit or create a free withdrawal. Restore connectivity/configuration and the
original journal to resolve holds; never delete a pending journal to unblock
the bank. If storage is permanently lost, reconcile against the retained
request and ledger records before any manual repair.
During removal, rebuild, host migration or launch cancellation, agent 1.9.0+
stops the game and drains its journals **before** deleting runtime files or
starting a replacement. It uses its host credential at
`POST /api/agent/bank/settle`; EC checks the assigned host, immutable instance,
unfinished teardown operation, server and original request identity. This
endpoint permits only commit/cancel, never new deposits or withdrawals. Journal
URLs must match that host's configured controller; the agent never follows a
journal URL. Corruption, a mismatched controller, failed acknowledgment or an
outage retains storage and leaves the operation unresolved for retry. An
interrupted rebuild uses the existing rebuild recovery workflow to retry with
a fresh operation after settlement. Do not manually erase its old directory.
The journal provides recovery from map/process restart. It cannot make live
game memory and SQLite one atomic transaction or preserve match points after a
machine/storage failure. Back up persistent storage and verify crash behavior
in the actual Windows/Plutonium deployment before rollout.
## History and validation
Player profiles show the shared balance, pending status and 20 newest ledger
entries, including signed amounts, resulting balances, server, time and match.
Accounts, request IDs and ledger entries have no cascading match/server foreign
keys and are not deleted by match retention or server deletion. Retained IDs
also keep old retries idempotent. Expired match links become plain historical
IDs. Include these tables in normal database backups.
Run `npx vitest run tests/unit/bank.test.ts`, then the usual lint, type checks,
tests and build. TWS supplies `node tools/check_bank.mjs` and
`python3 tools/test_plugin_bank.py`. Live validation must cover both games:
deposit/withdraw amounts and `all`, insufficient funds and caps, joining before
telemetry, `.help`, score HUD updates, death/respawn/disconnect during requests,
lost prepare/commit replies, controller outage, disk failures and map/process
restart with pending work. Confirm earned score and leaderboard behavior, and
verify that another server's credential and the telemetry credential fail.
+9
View File
@@ -1035,3 +1035,12 @@ game hosts. Run:
- Controller restart, agent restart and network loss before execution, during - Controller restart, agent restart and network loss before execution, during
execution and after execution but before acknowledgement. Verify the same execution and after execution but before acknowledgement. Verify the same
operation IDs and absence of duplicate containers/restarts. operation IDs and absence of duplicate containers/restarts.
### Bank journals during server teardown
Agent 1.9.0 adds `bank.settlement`: removal, rebuild, migration and creation
cancellation finish pending TWS bank decisions before deleting the old runtime.
An unavailable controller or corrupt journal leaves the operation unresolved
and preserves storage. Update all agents before enabling the
[player bank](bank.md). EC generates and injects bank credentials per instance;
no token map or manual server-secret provisioning is required.
+1
View File
@@ -37,6 +37,7 @@ export function routePermission(routeId: string | null, method: string): RoutePe
return 'admin'; return 'admin';
if (within(routeId, '/api/agent')) return 'machine'; if (within(routeId, '/api/agent')) return 'machine';
if (routeId === '/api/bank/[action]' && method === 'POST') return 'machine';
if (routeId === '/api/telemetry' && method === 'POST') return 'machine'; if (routeId === '/api/telemetry' && method === 'POST') return 'machine';
if (routeId === '/api/leaderboard' && reading) return 'public'; if (routeId === '/api/leaderboard' && reading) return 'public';
if ((routeId === '/login' || routeId === '/setup') && reading) return 'public'; if ((routeId === '/login' || routeId === '/setup') && reading) return 'public';
+46
View File
@@ -0,0 +1,46 @@
import { createHash, randomBytes } from 'node:crypto';
import { db } from '../db/index.js';
const digest = (value: string) => createHash('sha256').update(value).digest('hex');
/** Private controller state, reused only by the same immutable native instance. */
export function bankCredential(instanceId: string): string {
return db()
.transaction(() => {
const instance = db()
.prepare('SELECT server FROM native_instances WHERE id=? AND retired_at IS NULL')
.get(instanceId) as { server: string } | undefined;
if (!instance) throw new Error('Bank credentials require an active native instance.');
const saved = db()
.prepare('SELECT token FROM bank_credentials WHERE instance_id=?')
.get(instanceId) as { token: string } | undefined;
if (saved) return saved.token;
const token = 'ec_bank_' + randomBytes(32).toString('base64url');
db()
.prepare(
'INSERT INTO bank_credentials(instance_id,server,token,token_hash,created_at) VALUES (?,?,?,?,?)'
)
.run(instanceId, instance.server, token, digest(token), new Date().toISOString());
return token;
})
.immediate();
}
/** Retiring an instance immediately revokes its game credential, including reads. */
export function authenticateBank(
request: Request
): { ok: true; server: string; instanceId: string } | { ok: false; status: 401; code: string } {
const token = request.headers
.get('authorization')
?.match(/^Bearer (ec_bank_[A-Za-z0-9_-]{32,128})$/i)?.[1];
const matched = token
? (db()
.prepare(
`SELECT c.server, c.instance_id AS instanceId
FROM bank_credentials c JOIN native_instances n ON n.id=c.instance_id AND n.server=c.server
WHERE c.token_hash=? AND n.retired_at IS NULL`
)
.get(digest(token)) as { server: string; instanceId: string } | undefined)
: undefined;
return matched ? { ok: true, ...matched } : { ok: false, status: 401, code: 'unauthorized' };
}
+22
View File
@@ -0,0 +1,22 @@
import type Database from 'better-sqlite3';
export function migrateBankCredentials(handle: Database.Database): void {
handle.exec(`
CREATE TABLE IF NOT EXISTS bank_credentials (
instance_id TEXT PRIMARY KEY, server TEXT NOT NULL,
token TEXT NOT NULL, token_hash TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL
);
`);
if (
!(handle.pragma('table_info(bank_requests)') as { name: string }[]).some(
(column) => column.name === 'instance_id'
)
) {
handle.exec(`
ALTER TABLE bank_requests ADD COLUMN instance_id TEXT;
UPDATE bank_requests SET instance_id = (
SELECT id FROM native_instances WHERE server = bank_requests.server AND retired_at IS NULL
);
`);
}
}
+27
View File
@@ -0,0 +1,27 @@
import type Database from 'better-sqlite3';
/** No match/server foreign keys: money and idempotency outlive fleet/history cleanup. */
export function migrateBank(handle: Database.Database): void {
handle.exec(`
CREATE TABLE IF NOT EXISTS bank_accounts (
guid TEXT PRIMARY KEY,
balance INTEGER NOT NULL DEFAULT 0 CHECK (typeof(balance) = 'integer' AND balance BETWEEN 0 AND 1000000)
);
CREATE TABLE IF NOT EXISTS bank_requests (
server TEXT NOT NULL, request_id TEXT NOT NULL, guid TEXT NOT NULL,
fingerprint TEXT NOT NULL, operation TEXT NOT NULL CHECK (operation IN ('deposit','withdraw')),
match_id INTEGER, amount INTEGER NOT NULL DEFAULT 0,
state TEXT NOT NULL CHECK (state IN ('prepared','committed','cancelled','rejected')),
result TEXT NOT NULL, created_at TEXT NOT NULL, settled_at TEXT,
PRIMARY KEY(server, request_id)
);
CREATE UNIQUE INDEX IF NOT EXISTS bank_pending_guid ON bank_requests(guid) WHERE state = 'prepared';
CREATE TABLE IF NOT EXISTS bank_ledger (
id INTEGER PRIMARY KEY AUTOINCREMENT, guid TEXT NOT NULL,
server TEXT NOT NULL, request_id TEXT NOT NULL, match_id INTEGER,
amount INTEGER NOT NULL, balance INTEGER NOT NULL, created_at TEXT NOT NULL,
UNIQUE(server, request_id)
);
CREATE INDEX IF NOT EXISTS bank_ledger_guid ON bank_ledger(guid, id DESC);
`);
}
+5 -1
View File
@@ -18,6 +18,8 @@ import { migrateAuthSchema, migrateSuperViewerRole } from '../auth/schema.js';
import { migrateGameFileSchema } from '../game-files/schema.js'; import { migrateGameFileSchema } from '../game-files/schema.js';
import { migrateLeaderboardReports } from './leaderboard-reports.js'; import { migrateLeaderboardReports } from './leaderboard-reports.js';
import { migratePlayerScoreTotal } from './player-score-total.js'; import { migratePlayerScoreTotal } from './player-score-total.js';
import { migrateBank } from './bank.js';
import { migrateBankCredentials } from './bank-credentials.js';
import type { Migration } from './migration-runner.js'; import type { Migration } from './migration-runner.js';
// Append only: never renumber entries or change an applied migration to add an // Append only: never renumber entries or change an applied migration to add an
@@ -109,7 +111,9 @@ export const migrations: readonly Migration[] = [
version: 19, version: 19,
name: 'cumulative player score per match', name: 'cumulative player score per match',
run: migratePlayerScoreTotal run: migratePlayerScoreTotal
} },
{ version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank },
{ version: 21, name: 'instance-owned bank credentials', run: migrateBankCredentials }
]; ];
export const databaseVersion = migrations.length; export const databaseVersion = migrations.length;
+243
View File
@@ -0,0 +1,243 @@
import { db } from '../db/index.js';
export const BANK_HARD_LIMIT = 1_000_000;
export type BankAction = 'balance' | 'deposit' | 'withdraw' | 'commit' | 'cancel';
interface Transfer {
server: string;
guid: string;
match: string;
request_id: string;
operation: 'deposit' | 'withdraw';
amount: number | 'all';
capacity: number;
}
interface Result {
ok: boolean;
state: string;
code: string;
request_id: string;
amount: number;
balance: number;
}
interface Saved {
instance_id: string | null;
fingerprint: string;
state: string;
amount: number;
match_id: number | null;
result: string;
}
const token = (value: unknown, limit: number): value is string =>
typeof value === 'string' &&
value.length > 0 &&
value.length <= limit &&
/^[A-Za-z0-9_.:-]+$/.test(value);
const integer = (value: unknown, min = 0): value is number =>
Number.isSafeInteger(value) && Number(value) >= min && Number(value) <= BANK_HARD_LIMIT;
export function bankMaximum(): number {
const value = Number(process.env.EC_BANK_MAX_BALANCE ?? 250_000);
if (!integer(value, 1))
throw new Error('EC_BANK_MAX_BALANCE must be an integer from 1 to 1000000');
return value;
}
export function bankProfile(guid: string) {
const handle = db();
const account = handle.prepare('SELECT balance FROM bank_accounts WHERE guid = ?').get(guid) as
{ balance: number } | undefined;
const pending = handle
.prepare(
"SELECT operation, amount FROM bank_requests WHERE guid = ? AND state = 'prepared'"
)
.get(guid) as { operation: string; amount: number } | undefined;
const entries = handle
.prepare(
`SELECT l.id, l.server, l.amount, l.balance, l.created_at AS createdAt,
l.match_id AS matchId, CASE WHEN m.id IS NOT NULL THEN 1 ELSE 0 END AS matchAvailable
FROM bank_ledger l LEFT JOIN leaderboard_matches m ON m.id = l.match_id
WHERE l.guid = ? ORDER BY l.id DESC LIMIT 20`
)
.all(guid) as {
id: number;
server: string;
amount: number;
balance: number;
createdAt: string;
matchId: number | null;
matchAvailable: number;
}[];
const balance = account?.balance ?? 0;
return {
balance,
available: balance - (pending?.operation === 'withdraw' ? pending.amount : 0),
pending: Boolean(pending),
entries
};
}
/** One immediate SQLite transaction serializes limits, reservations and ledger updates. */
export function bankRequest(
action: BankAction,
value: unknown,
authenticatedServer: string,
instanceId: string
): { status: number; body: Result } {
const result = (status: number, code: string, request_id = '') => ({
status,
body: { ok: false, state: 'rejected', code, request_id, amount: 0, balance: 0 }
});
if (!value || typeof value !== 'object' || Array.isArray(value))
return result(400, 'invalid_request');
const input = value as Transfer;
if (input.server !== authenticatedServer) return result(403, 'wrong_server');
if (!token(input.guid, 64)) return result(400, 'invalid_request');
if (action === 'balance') {
const profile = bankProfile(input.guid);
return {
status: 200,
body: {
ok: true,
state: 'balance',
code: profile.pending ? 'pending' : 'ok',
request_id: '',
amount: profile.available,
balance: profile.balance
}
};
}
if (
!token(input.request_id, 64) ||
!token(input.match, 64) ||
!['deposit', 'withdraw'].includes(input.operation) ||
!(integer(input.amount, 1) || (input.operation === 'withdraw' && input.amount === 'all')) ||
!integer(input.capacity) ||
((action === 'deposit' || action === 'withdraw') && action !== input.operation)
)
return result(400, 'invalid_request');
const { server, guid, request_id, operation, match, amount, capacity } = input;
const fingerprint = JSON.stringify({ guid, operation, match, amount, capacity });
const handle = db();
return handle
.transaction(() => {
const saved = handle
.prepare('SELECT * FROM bank_requests WHERE server = ? AND request_id = ?')
.get(server, request_id) as Saved | undefined;
if (saved && (saved.instance_id !== instanceId || saved.fingerprint !== fingerprint))
return result(409, 'request_conflict', request_id);
const balance =
(
handle.prepare('SELECT balance FROM bank_accounts WHERE guid = ?').get(guid) as
{ balance: number } | undefined
)?.balance ?? 0;
const now = new Date().toISOString();
const reply = (
state: string,
code: string,
transferred = 0,
resulting = balance
): Result => ({
ok: state !== 'rejected',
state,
code,
request_id,
amount: transferred,
balance: resulting
});
const insert = (body: Result, matchId: number | null = null) => {
handle
.prepare(
`INSERT INTO bank_requests(server,request_id,guid,fingerprint,operation,match_id,amount,state,result,created_at,instance_id)
VALUES (?,?,?,?,?,?,?,?,?,?,?)`
)
.run(
server,
request_id,
guid,
fingerprint,
operation,
matchId,
body.amount,
body.state,
JSON.stringify(body),
now,
instanceId
);
return { status: 200, body };
};
if (action === 'cancel') {
// Tombstones win even when an earlier prepare is still in transit.
if (!saved) return insert(reply('cancelled', 'cancelled'));
if (saved.state !== 'prepared')
return { status: 200, body: JSON.parse(saved.result) as Result };
const body = reply('cancelled', 'cancelled');
handle
.prepare(
'UPDATE bank_requests SET state = ?, result = ?, settled_at = ? WHERE server = ? AND request_id = ?'
)
.run(body.state, JSON.stringify(body), now, server, request_id);
return { status: 200, body };
}
if (action === 'commit') {
if (!saved) return result(409, 'not_prepared', request_id);
if (saved.state !== 'prepared')
return { status: 200, body: JSON.parse(saved.result) as Result };
const delta = operation === 'deposit' ? saved.amount : -saved.amount;
const resulting = balance + delta;
handle
.prepare(
`INSERT INTO bank_accounts(guid,balance) VALUES (?,?) ON CONFLICT(guid) DO UPDATE SET balance = excluded.balance`
)
.run(guid, resulting);
handle
.prepare(
`INSERT INTO bank_ledger(guid,server,request_id,match_id,amount,balance,created_at) VALUES (?,?,?,?,?,?,?)`
)
.run(guid, server, request_id, saved.match_id, delta, resulting, now);
const body = reply('committed', 'ok', saved.amount, resulting);
handle
.prepare(
'UPDATE bank_requests SET state = ?, result = ?, settled_at = ? WHERE server = ? AND request_id = ?'
)
.run(body.state, JSON.stringify(body), now, server, request_id);
return { status: 200, body };
}
if (saved) return { status: 200, body: JSON.parse(saved.result) as Result };
if (
!handle
.prepare(
'SELECT 1 FROM native_instances WHERE id=? AND server=? AND retired_at IS NULL'
)
.get(instanceId, server)
)
return result(403, 'retired_instance', request_id);
const member = handle
.prepare(
`SELECT m.id FROM leaderboard_matches m
JOIN servers s ON s.name = m.server AND s.destroyed_at IS NULL
JOIN leaderboard_match_players p ON p.match_id = m.id AND p.guid = ? AND p.present = 1
WHERE m.server = ? AND m.reporter_match_key = ? AND m.status = 'active'
AND m.id = (SELECT MAX(id) FROM leaderboard_matches WHERE server = ?)`
)
.get(guid, server, match, server) as { id: number } | undefined;
if (!member) return insert(reply('rejected', 'not_in_match'));
if (
handle
.prepare("SELECT 1 FROM bank_requests WHERE guid = ? AND state = 'prepared'")
.get(guid)
)
return insert(reply('rejected', 'pending'), member.id);
const requested = amount === 'all' ? balance : amount;
let code = 'ok';
if (requested === 0) code = 'empty';
else if (operation === 'withdraw' && requested > balance) code = 'insufficient_balance';
else if (operation === 'withdraw' && requested > capacity) code = 'points_limit';
else if (operation === 'deposit' && balance + requested > bankMaximum())
code = 'bank_limit';
return insert(
reply(code === 'ok' ? 'prepared' : 'rejected', code, code === 'ok' ? requested : 0),
member.id
);
})
.immediate();
}
@@ -13,6 +13,7 @@ export interface NativeLaunchInput {
instance: NativeInstance; instance: NativeInstance;
serverKey: string; serverKey: string;
rconPassword: string; rconPassword: string;
bankToken: string;
rebuild: boolean; rebuild: boolean;
safetyOverride?: boolean; safetyOverride?: boolean;
unhealthyRecovery?: boolean; unhealthyRecovery?: boolean;
+15 -1
View File
@@ -71,6 +71,19 @@ parentPort!.on(
}; };
})(); })();
const instance = input.instance; const instance = input.instance;
// Credential files travel only in this host's private operation payload, never
// shared recipes, promoted files, or the content-addressed download store.
const bankPath = posix.join(
relDestForKind('config', recipe.game),
'electric_cherry_bank.cfg'
);
if (
recipe.files.some(
(file) =>
posix.join(relDestForKind(file.kind, recipe.game), file.path) === bankPath
)
)
throw new Error('electric_cherry_bank.cfg is reserved for instance credentials.');
const payload = { const payload = {
name: instance.server, name: instance.server,
game: recipe.game, game: recipe.game,
@@ -79,13 +92,14 @@ parentPort!.on(
port: instance.port, port: instance.port,
bindAddress: instance.bind_address, bindAddress: instance.bind_address,
rconPassword: password, rconPassword: password,
bank: { token: input.bankToken, configPath: bankPath },
env: { env: {
PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_SERVER_NAME: instance.server,
PLUTAINER_PORT: String(instance.port), PLUTAINER_PORT: String(instance.port),
PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp', PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp',
PLUTAINER_CONFIG_FILE: recipe.configFile, PLUTAINER_CONFIG_FILE: recipe.configFile,
PLUTAINER_MAP_ROTATE: 'false', PLUTAINER_MAP_ROTATE: 'false',
PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +exec electric_cherry_bank.cfg +set tws_server_id ${instance.server} +map_rotate`,
PLUTO_SERVER_KEY: input.serverKey PLUTO_SERVER_KEY: input.serverKey
}, },
files: recipe.files.map((file) => ({ files: recipe.files.map((file) => ({
+2
View File
@@ -12,6 +12,7 @@ import { isIP } from 'node:net';
import { setImmediate as yieldToController } from 'node:timers/promises'; import { setImmediate as yieldToController } from 'node:timers/promises';
import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js'; import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
import { randomBytes, randomUUID } from 'node:crypto'; import { randomBytes, randomUUID } from 'node:crypto';
import { bankCredential } from '../auth/bank.js';
import { posix } from 'node:path'; import { posix } from 'node:path';
import { db } from '../db/index.js'; import { db } from '../db/index.js';
import { unsettledCancelledCommand } from './cancellation.js'; import { unsettledCancelledCommand } from './cancellation.js';
@@ -1936,6 +1937,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
instance, instance,
serverKey: preparedKey, serverKey: preparedKey,
rconPassword: preparedPassword, rconPassword: preparedPassword,
bankToken: bankCredential(instance.id),
rebuild: job.action === 'rebuild' && !migration, rebuild: job.action === 'rebuild' && !migration,
safetyOverride: overrideSafety, safetyOverride: overrideSafety,
unhealthyRecovery: host.inventory.capabilities?.includes( unhealthyRecovery: host.inventory.capabilities?.includes(
@@ -0,0 +1,28 @@
import { error, json } from '@sveltejs/kit';
import { authenticateAgent, objectBody } from '$lib/server/hosts/http.js';
import { db } from '$lib/server/db/index.js';
import { bankRequest } from '$lib/server/fleet/bank.js';
/** An assigned teardown may finish durable decisions, never prepare new spending. */
export async function POST({ request }: { request: Request }) {
const host = authenticateAgent(request);
const body = await objectBody(request, 8192);
if (typeof body.operationId !== 'string' || !['commit', 'cancel'].includes(String(body.action)))
error(400, 'Invalid bank settlement.');
const scope = db()
.prepare(
`SELECT n.id, n.server FROM ec_host_operations o
JOIN native_instances n ON n.id=o.instance_id
WHERE o.id=? AND o.host_id=? AND o.kind IN ('instance.remove','instance.rebuild','instance.cancel-create')
AND o.state IN ('pending','accepted','running','unknown')`
)
.get(body.operationId, host.id) as { id: string; server: string } | undefined;
if (!scope) error(403, 'Bank settlement requires an assigned, unfinished teardown.');
const result = bankRequest(
body.action as 'commit' | 'cancel',
body.transfer,
scope.server,
scope.id
);
return json(result.body, { status: result.status, headers: { 'Cache-Control': 'no-store' } });
}
+40
View File
@@ -0,0 +1,40 @@
import { json, isHttpError } from '@sveltejs/kit';
import { authenticateBank } from '$lib/server/auth/bank.js';
import { bankRequest, type BankAction } from '$lib/server/fleet/bank.js';
import { jsonBody } from '$lib/server/request-body.js';
export async function POST({ request, params }: { request: Request; params: { action: string } }) {
const headers = { 'Cache-Control': 'no-store' };
const authentication = authenticateBank(request);
if (!authentication.ok)
return json(
{ ok: false, code: authentication.code },
{ status: authentication.status, headers }
);
if (!['balance', 'deposit', 'withdraw', 'commit', 'cancel'].includes(params.action))
return json({ ok: false, code: 'invalid_action' }, { status: 404, headers });
if (
request.headers.get('content-type')?.split(';')[0].trim().toLowerCase() !==
'application/json'
)
return json({ ok: false, code: 'invalid_content_type' }, { status: 415, headers });
let value: unknown;
try {
value = await jsonBody(request, 4096);
} catch (cause) {
if (isHttpError(cause, 413)) throw cause;
return json({ ok: false, code: 'invalid_request' }, { status: 400, headers });
}
try {
const { body, status } = bankRequest(
params.action as BankAction,
value,
authentication.server,
authentication.instanceId
);
return json(body, { status, headers });
} catch {
console.error('Bank transaction failed; request may be retried with the same identity.');
return json({ ok: false, code: 'unavailable' }, { status: 503, headers });
}
}
@@ -1,3 +1,4 @@
import { bankProfile } from '$lib/server/fleet/bank.js';
import { error } from '@sveltejs/kit'; import { error } from '@sveltejs/kit';
import { playerReport } from '$lib/server/reports/index.js'; import { playerReport } from '$lib/server/reports/index.js';
import { isLeaderboardMetric } from '$lib/server/fleet/leaderboard.js'; import { isLeaderboardMetric } from '$lib/server/fleet/leaderboard.js';
@@ -38,6 +39,7 @@ export async function load({ params, url }: { params: { guid: string }; url: URL
if (!profile) error(404, 'Player not found'); if (!profile) error(404, 'Player not found');
return { return {
profile, profile,
bank: bankProfile(params.guid),
returnTo: leaderboardReturnHref(url.searchParams.get('returnTo')), returnTo: leaderboardReturnHref(url.searchParams.get('returnTo')),
scope: { game: game ?? '', map: map ?? '', players: players ?? '', metric, entry } scope: { game: game ?? '', map: map ?? '', players: players ?? '', metric, entry }
}; };
@@ -1,4 +1,5 @@
<script lang="ts"> <script lang="ts">
import type { bankProfile } from '$lib/server/fleet/bank.js';
import Select from '$ui/components/primitives/Select.svelte'; import Select from '$ui/components/primitives/Select.svelte';
import Field from '$ui/components/primitives/Field.svelte'; import Field from '$ui/components/primitives/Field.svelte';
import RelativeTime from '$ui/components/RelativeTime.svelte'; import RelativeTime from '$ui/components/RelativeTime.svelte';
@@ -43,6 +44,7 @@
}: { }: {
data: { data: {
profile: PlayerProfile; profile: PlayerProfile;
bank: ReturnType<typeof bankProfile>;
returnTo: string; returnTo: string;
scope: { scope: {
game: string; game: string;
@@ -162,6 +164,45 @@
description="Stats appear after a game's end report arrives, provided this player was observed at round 2 or later. Games without an end report do not contribute." description="Stats appear after a game's end report arrives, provided this player was observed at round 2 or later. Games without an end report do not contribute."
/> />
{/if} {/if}
<section aria-labelledby="bank-title">
<h2 id="bank-title">Bank</h2>
<p>
<strong>{number.format(data.bank.balance)} points</strong> shared across Black Ops I and II.
</p>
{#if data.bank.pending}<p>
A transfer is pending. Available: {number.format(data.bank.available)} points.
</p>{/if}
{#if data.bank.entries.length}
<div class="table-scroll">
<table>
<caption>Recent bank transfers</caption>
<thead
><tr
><th scope="col">Time</th><th scope="col">Server</th><th scope="col"
>Transfer</th
><th scope="col">Balance</th><th scope="col">Match</th></tr
></thead
>
<tbody
>{#each data.bank.entries as entry (entry.id)}
<tr
><td>{$timeFormat.dateTime(entry.createdAt)}</td><td
>{serverTitle(entry.server)}</td
><td>{entry.amount > 0 ? '+' : ''}{number.format(entry.amount)}</td
><td>{number.format(entry.balance)}</td><td
>{#if entry.matchAvailable}<a href={`/matches/${entry.matchId}`}
>#{entry.matchId}</a
>{:else}{entry.matchId
? `#${entry.matchId} (history expired)`
: '—'}{/if}</td
></tr
>
{/each}</tbody
>
</table>
</div>
{:else}<p>No bank transfers yet.</p>{/if}
</section>
<nav class="jump-links" aria-label="Player sections"> <nav class="jump-links" aria-label="Player sections">
<a href="#overview">Overview</a><a href="#maps-title">Maps</a><a href="#history-title" <a href="#overview">Overview</a><a href="#maps-title">Maps</a><a href="#history-title"
>Matches</a >Matches</a
+96 -6
View File
@@ -35,7 +35,7 @@ import urllib.parse
import urllib.request import urllib.request
import uuid import uuid
VERSION = "1.8.7" VERSION = "1.9.0"
LABEL = "io.electric-cherry" LABEL = "io.electric-cherry"
MAX_FILE = 32 * 1024 * 1024 MAX_FILE = 32 * 1024 * 1024
MANIFEST_MAX_ENTRIES = 10000 MANIFEST_MAX_ENTRIES = 10000
@@ -1503,7 +1503,7 @@ class Agent:
if line.startswith("PRETTY_NAME="): if line.startswith("PRETTY_NAME="):
os_name = line.split("=", 1)[1].strip('"') os_name = line.split("=", 1)[1].strip('"')
update = self.update_inventory() update = self.update_inventory()
capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) capabilities = ["library.remove", "plutonium.cache", "instance.cancel-create", "instance.safety-override", "instance.unhealthy-recovery", "instance.autoscaling-remove", "operations.read-lane", "operations.health-lane", "rcon.status-batch", "instance.heartbeat-state", "files.manifest", "files.blobs", "logs.read-offset", "bank.settlement"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
plutonium_cache = self.plutonium_cache_inventory() plutonium_cache = self.plutonium_cache_inventory()
return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "agentLanes": self.lane_inventory(), "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, **resources, **({"cpuUsagePercent": cpu_usage} if cpu_usage is not None else {}), "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "preparedRuntimeImages": sorted(set(prepared_images)), **({"plutoniumCache": plutonium_cache} if plutonium_cache else {}), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
@@ -1657,7 +1657,7 @@ class Agent:
# record. Whenever preparation was recorded, enforce that exact image. # record. Whenever preparation was recorded, enforce that exact image.
if image in prepared and prepared[image] != image_id: if image in prepared and prepared[image] != image_id:
raise AgentError("Prepared runtime image has changed. An admin must prepare it again before launching.") raise AgentError("Prepared runtime image has changed. An admin must prepare it again before launching.")
files = payload.get("files", []) files = self.initial_files(payload)
if not isinstance(files, list) or len(files) > 10000: if not isinstance(files, list) or len(files) > 10000:
raise Rejected("Invalid initial file list") raise Rejected("Invalid initial file list")
total, paths = 0, set() total, paths = 0, set()
@@ -1911,7 +1911,7 @@ class Agent:
self.operation_phase = "seeding-runtime" self.operation_phase = "seeding-runtime"
self.seed_plutonium(app, image) self.seed_plutonium(app, image)
self.operation_phase = "writing-create-files" self.operation_phase = "writing-create-files"
self.write_initial_files(app, payload.get("files", [])) self.write_initial_files(app, self.initial_files(payload))
self.operation_phase = "persisting-create-metadata" self.operation_phase = "persisting-create-metadata"
atomic_json(directory / "spec.json", payload) atomic_json(directory / "spec.json", payload)
env_path = directory / "environment" env_path = directory / "environment"
@@ -1944,6 +1944,25 @@ class Agent:
if self.executing_id and self.executing_id in self.cancelled_creations: if self.executing_id and self.executing_id in self.cancelled_creations:
raise AgentError("Creation cancelled; waiting for instance cleanup") raise AgentError("Creation cancelled; waiting for instance cleanup")
def initial_files(self, payload):
files = payload.get("files", [])
if not isinstance(files, list):
raise Rejected("Invalid initial file list")
bank = payload.get("bank")
if bank is None:
return files
if not isinstance(bank, dict) or not isinstance(bank.get("token"), str) or not re.fullmatch(r"ec_bank_[A-Za-z0-9_-]{32,128}", bank["token"]):
raise Rejected("Invalid instance bank credential")
path = bank.get("configPath")
scoped(self.root / "agent" / "validation", path)
if PurePosixPath(path).name != "electric_cherry_bank.cfg":
raise Rejected("Invalid private bank config path")
controller = self.config["controller"].rstrip("/")
if any(char in controller for char in ('"', "\\", "\n", "\r", "\x00")):
raise Rejected("Invalid bank controller URL")
content = f'set tws_bank_token "{bank["token"]}"\nset tws_bank_url "{controller}/api/bank"\n'.encode()
return files + [{"path": path, "contentBase64": base64.b64encode(content).decode()}]
def write_initial_files(self, root, files): def write_initial_files(self, root, files):
if not isinstance(files, list) or len(files) > 10000: if not isinstance(files, list) or len(files) > 10000:
raise AgentError("Invalid file list") raise AgentError("Invalid file list")
@@ -1992,6 +2011,72 @@ class Agent:
if (time.monotonic() - started) * 1000 >= ttl: if (time.monotonic() - started) * 1000 >= ttl:
raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation") raise AutoscalingInvalidated("Autoscaling removal authorization expired before mutation")
def settle_bank(self, instance_id, generation, operation_id=None):
"""Drain stopped runtime journals before discarding any generation's storage.
Use only our configured controller and host credential. Never follow an
endpoint or symlink supplied by a game journal. Lost replies are retried
with the original request identity; files remain until teardown succeeds.
"""
app = scoped(self.instance_dir(instance_id, generation), "app")
if not app.exists():
return
if self.inspect(instance_id, generation)["running"]:
raise AgentError("Bank settlement requires a stopped instance")
operation_id = operation_id or self.executing_id
visited = 0
deadline = time.monotonic() + 30
journals = []
def unreadable(_error):
raise AgentError("Bank journal discovery failed; runtime storage retained")
for directory, directories, files in os.walk(app, followlinks=False, onerror=unreadable):
relative = Path(directory).relative_to(app)
inside_bank = "tws-bank" in relative.parts
visited += 1
if visited > 100000 or time.monotonic() > deadline:
raise AgentError("Bank journal discovery limit exceeded; runtime storage retained")
for name in directories + files:
if (inside_bank or name == "tws-bank") and (Path(directory) / name).is_symlink():
raise AgentError("Bank journal paths must not be symlinks")
if not inside_bank:
continue
for name in files:
if name.endswith(".json"):
journals.append((relative / name).as_posix())
if len(journals) > 4096:
raise AgentError("Too many bank journals; runtime storage retained")
for path in sorted(journals):
try:
raw = managed_read(app, path)
except Rejected:
raise AgentError("Bank journal cannot be read; runtime storage retained") from None
if raw is None or len(raw) > 16 * 1024 * 1024:
raise AgentError("Bank journal is unavailable or oversized")
try:
journal = json.loads(raw)
reports = journal["reports"]
if journal["endpoint"].rstrip("/") != self.config["controller"].rstrip("/") + "/api/bank":
raise AgentError("Bank journal belongs to a different controller; runtime storage retained")
if type(journal["version"]) is not int or journal["version"] != 1 or not isinstance(reports, list) or len(reports) > 4096:
raise ValueError()
# Controller independently binds this operation to the instance/server.
for report in reports:
if not isinstance(report, str) or len(report.encode()) > 20000:
raise ValueError()
transfer = json.loads(report)
action = transfer["decision"]
if action not in ("commit", "cancel") or transfer["server"] != journal["server"] or not operation_id:
raise ValueError()
response = self.request("POST", "/api/agent/bank/settle", {
"operationId": operation_id, "action": action, "transfer": transfer})
expected = ("committed",) if action == "commit" else ("cancelled", "rejected")
if not isinstance(response, dict) or response.get("request_id") != transfer["request_id"] or response.get("state") not in expected:
raise AgentError("Bank settlement was not acknowledged; runtime storage retained")
except (ValueError, TypeError, KeyError, AttributeError):
raise AgentError("Bank journal is invalid; runtime storage retained") from None
def lifecycle(self, kind, instance_id, generation, payload): def lifecycle(self, kind, instance_id, generation, payload):
unhealthy_recovery = payload.get("unhealthyRecovery") is True unhealthy_recovery = payload.get("unhealthyRecovery") is True
if unhealthy_recovery and (kind not in ("instance.restart", "instance.rebuild") or payload.get("stageId") or "autoscaling" in payload): if unhealthy_recovery and (kind not in ("instance.restart", "instance.rebuild") or payload.get("stageId") or "autoscaling" in payload):
@@ -2043,6 +2128,7 @@ class Agent:
if health["exists"]: if health["exists"]:
self.docker.call("container", "stop", "--time", "30", name) self.docker.call("container", "stop", "--time", "30", name)
self.docker.call("container", "rm", name) self.docker.call("container", "rm", name)
self.settle_bank(instance_id, generation)
if kind == "instance.rebuild": if kind == "instance.rebuild":
result = self.create(instance_id, next_generation, spec) result = self.create(instance_id, next_generation, spec)
self.operation_phase = "removing-old-instance" self.operation_phase = "removing-old-instance"
@@ -2062,7 +2148,7 @@ class Agent:
raise AgentError("Unsupported instance operation") raise AgentError("Unsupported instance operation")
return {**self.inspect(instance_id, generation), "before": before} return {**self.inspect(instance_id, generation), "before": before}
def cancel_creation(self, instance_id, generation, payload): def cancel_creation(self, instance_id, generation, payload, operation_id=None):
"""Explicit launch cancellation, fenced to its immutable creation identity. """Explicit launch cancellation, fenced to its immutable creation identity.
The operation loop serializes this with create. Once delivered, removal The operation loop serializes this with create. Once delivered, removal
@@ -2106,6 +2192,7 @@ class Agent:
self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, target_generation)) self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, target_generation))
if self.inspect(instance_id, target_generation)["exists"]: if self.inspect(instance_id, target_generation)["exists"]:
raise AgentError("Cancelled creation container still exists") raise AgentError("Cancelled creation container still exists")
self.settle_bank(instance_id, target_generation, operation_id)
directory = self.instance_dir(instance_id, target_generation) directory = self.instance_dir(instance_id, target_generation)
if directory.exists(): if directory.exists():
shutil.rmtree(directory) shutil.rmtree(directory)
@@ -2747,6 +2834,7 @@ class Agent:
if not unchanged_generation() or not both_missing(): if not unchanged_generation() or not both_missing():
return row return row
self.settle_bank(instance_id, generation, operation["id"])
# Preparation can leave runtime/configuration files before create's # Preparation can leave runtime/configuration files before create's
# checkpoint. A fresh operation must not inherit those partial files. # checkpoint. A fresh operation must not inherit those partial files.
expected_spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")} expected_spec = payload.get("spec") or {key: value for key, value in payload.items() if key not in ("nextGeneration", "newGeneration", "unhealthyRecovery")}
@@ -2793,7 +2881,7 @@ class Agent:
if kind == "instance.remove" and "autoscaling" in payload and not checkpoint: if kind == "instance.remove" and "autoscaling" in payload and not checkpoint:
return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision") return self.journal.update(operation["id"], "failed", {"autoscalingInvalidated": True}, error="Interrupted autoscaling removal requires a new decision")
if kind == "instance.cancel-create": if kind == "instance.cancel-create":
return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload)) return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload, operation["id"]))
if kind == "instance.create": if kind == "instance.create":
marker = self.instance_dir(instance_id, generation).parent / "current.json" marker = self.instance_dir(instance_id, generation).parent / "current.json"
if marker.exists(): if marker.exists():
@@ -2885,6 +2973,7 @@ class Agent:
directory = self.instance_dir(instance_id, observed_generation) directory = self.instance_dir(instance_id, observed_generation)
atomic_json(directory.parent / "current.json", {"generation": observed_generation}) atomic_json(directory.parent / "current.json", {"generation": observed_generation})
if kind == "instance.rebuild" and not self.inspect(instance_id, generation)["exists"]: if kind == "instance.rebuild" and not self.inspect(instance_id, generation)["exists"]:
self.settle_bank(instance_id, generation, operation["id"])
old = self.instance_dir(instance_id, generation) old = self.instance_dir(instance_id, generation)
if old.exists(): if old.exists():
shutil.rmtree(old) shutil.rmtree(old)
@@ -2899,6 +2988,7 @@ class Agent:
elif kind == "instance.remove": elif kind == "instance.remove":
proven = not health["exists"] proven = not health["exists"]
if proven: if proven:
self.settle_bank(instance_id, generation, operation["id"])
directory = self.instance_dir(instance_id, generation) directory = self.instance_dir(instance_id, generation)
if directory.exists(): if directory.exists():
shutil.rmtree(directory) shutil.rmtree(directory)
+2 -2
View File
@@ -1,8 +1,8 @@
{ {
"version": "1.8.7", "version": "1.9.0",
"protocolVersion": 1, "protocolVersion": 1,
"platform": "debian-amd64", "platform": "debian-amd64",
"sha256": "e773c907141c97eab2055b354306973b248a67109a51337f1f1d76d80aaa8701", "sha256": "07f91ba8780c1e2b0a9c0ab3534c1731d4246d3f5d84fece14691c930dcb5e90",
"updaterProtocolVersion": 2, "updaterProtocolVersion": 2,
"updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb", "updaterSha256": "a9d7f96b87d0ec74d8a0f476dff6e3d2a7a0bc454a93a51d7b57de00c9ce23cb",
"installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231" "installerSha256": "384d5e397a51d19cb9fc9ed8793b4314353f726e217030246050859c0bc18231"
+5 -5
View File
@@ -1,8 +1,8 @@
-----BEGIN PGP SIGNATURE----- -----BEGIN PGP SIGNATURE-----
iJEEABYKADkWIQSpocb6VBfQzvdIh9HF6fdSNniN+gUCasmr1RsUgAAAAAAEAA5t iJEEABYKADkWIQSpocb6VBfQzvdIh9HF6fdSNniN+gUCas2ANxsUgAAAAAAEAA5t
YW51MiwyLjUrMS4xMiwwLDMACgkQxen3UjZ4jfprZQD+PPqrtEYWTImhdy1eFAfv YW51MiwyLjUrMS4xMiwwLDMACgkQxen3UjZ4jfotJAD/df3qwKWk45tYFghvCJZm
k5XD1IaFOgSvcsL9P0tj69IBAIRGt3zqWw9E8vFeyZXz58GfnscyFI6ws5ln+l3h viIxpqvNvCpxkvjgicV+d00BAIJRbVswzE46KRMimu8P61re0b2ppqEEfRmQc2N6
i2UP 3YkE
=Er1r =cojL
-----END PGP SIGNATURE----- -----END PGP SIGNATURE-----
+353
View File
@@ -0,0 +1,353 @@
import { randomUUID } from 'node:crypto';
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { routePermission } from '../../src/lib/server/auth/access.js';
let fixture: Fixture;
const server = 't5_zombie_theater_1';
const other = 't6_zm_transit_1';
let secret: string;
let otherSecret: string;
let instanceId: string;
let hostId: string;
let hostToken: string;
let serial = 0;
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([server, other]);
vi.stubEnv('EC_TELEMETRY_TOKEN', 'telemetry_' + 't'.repeat(40));
vi.stubEnv('EC_BANK_MAX_BALANCE', '250000');
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
await seedServerRegistry();
const { db } = await import('../../src/lib/server/db/index.js');
const { bankCredential } = await import('../../src/lib/server/auth/bank.js');
const { createHost, createEnrollment, enrollHost } =
await import('../../src/lib/server/hosts/index.js');
hostId = createHost({ name: 'Bank host' }).id;
hostToken = enrollHost(createEnrollment(hostId).token, {}).credential;
for (const source of [server, other]) {
const id = randomUUID();
db()
.prepare(
`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES (?,?,?,'library','image',28960,?)`
)
.run(id, source, hostId, new Date().toISOString());
if (source === server) {
instanceId = id;
secret = bankCredential(id);
} else otherSecret = bankCredential(id);
saveTelemetry({
server: source,
map: source === server ? 'zombie_theater' : 'zm_transit',
match: 'match-one',
seq: 1,
event: 'game_start',
players: [{ guid: 'ace', name: 'Ace' }],
receivedAt: new Date().toISOString()
});
}
});
afterEach(async () => {
await fixture.cleanup();
vi.unstubAllEnvs();
});
const transfer = (operation = 'deposit', amount: number | string = 2000, source = server) => ({
server: source,
guid: 'ace',
match: 'match-one',
request_id: `request-${++serial}`,
operation,
amount,
capacity: 1000000
});
async function post(action: string, body: unknown, credential = secret) {
const { POST } = await import('../../src/routes/api/bank/[action]/+server.js');
const response = await POST({
params: { action },
request: new Request(`https://example.test/api/bank/${action}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${credential}` },
body: JSON.stringify(body)
})
});
expect(response.headers.get('cache-control')).toBe('no-store');
return { status: response.status, ...(await response.json()) };
}
async function fund(amount = 2000) {
const data = transfer('deposit', amount);
expect(await post('deposit', data)).toMatchObject({ state: 'prepared', amount });
expect(await post('commit', data)).toMatchObject({ state: 'committed', balance: amount });
return data;
}
it('routes only bank POSTs as machine requests, authenticating before reading bodies', async () => {
expect(routePermission('/api/bank/[action]', 'POST')).toBe('machine');
expect(routePermission('/api/bank/[action]', 'GET')).toBe('viewer');
const { POST } = await import('../../src/routes/api/bank/[action]/+server.js');
for (const credential of ['', process.env.EC_TELEMETRY_TOKEN!, otherSecret]) {
const request = new Request('https://test/api/bank/deposit', {
method: 'POST',
headers: { Authorization: `Bearer ${credential}` },
body: 'invalid JSON'
});
if (credential === otherSecret) continue;
expect((await POST({ request, params: { action: 'deposit' } })).status).toBe(401);
expect(request.bodyUsed).toBe(false);
}
expect(await post('deposit', transfer(), otherSecret)).toMatchObject({
status: 403,
code: 'wrong_server'
});
});
it('owns generated secrets by immutable instance, preserving rebuilds and revoking retired names', async () => {
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const { bankCredential } = await import('../../src/lib/server/auth/bank.js');
const data = transfer();
await post('deposit', data);
expect(secret).toMatch(/^ec_bank_[A-Za-z0-9_-]{43}$/);
expect(otherSecret).not.toBe(secret);
closeDb();
db().prepare('UPDATE native_instances SET generation=2 WHERE id=?').run(instanceId);
expect(bankCredential(instanceId)).toBe(secret);
db()
.prepare('UPDATE native_instances SET retired_at=? WHERE id=?')
.run(new Date().toISOString(), instanceId);
expect(await post('balance', { server, guid: 'ace' })).toMatchObject({ status: 401 });
expect(() => bankCredential(instanceId)).toThrow('active native instance');
const next = randomUUID();
db()
.prepare(
`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES (?,?,?,'library','image',28960,?)`
)
.run(next, server, hostId, new Date().toISOString());
const replacement = bankCredential(next);
const { migrateBankCredentials } = await import('../../src/lib/server/db/bank-credentials.js');
migrateBankCredentials(db()); // A legacy migration rerun must not rebind old requests.
expect(replacement).not.toBe(secret);
expect(await post('commit', data, replacement)).toMatchObject({
status: 409,
code: 'request_conflict'
});
expect(await post('cancel', data, replacement)).toMatchObject({ status: 409 });
});
it('settles journals only for the host assigned to an unfinished teardown, even after retirement', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { queueHostOperation } = await import('../../src/lib/server/hosts/index.js');
const { POST } = await import('../../src/routes/api/agent/bank/settle/+server.js');
const data = transfer();
await post('deposit', data);
const removal = queueHostOperation(
hostId,
'instance.remove',
{},
{ instanceId, generation: 1 }
);
db()
.prepare('UPDATE native_instances SET retired_at=? WHERE id=?')
.run(new Date().toISOString(), instanceId);
const settle = (
action = 'commit',
operationId = removal.id,
credential = hostToken,
value = data
) =>
POST({
request: new Request('https://test/api/agent/bank/settle', {
method: 'POST',
headers: { Authorization: `Bearer ${credential}` },
body: JSON.stringify({ operationId, action, transfer: value })
})
});
await expect(settle('commit', removal.id, secret)).rejects.toMatchObject({ status: 401 });
const { createHost, createEnrollment, enrollHost } =
await import('../../src/lib/server/hosts/index.js');
const stranger = createHost({ name: 'Unassigned host' });
const strangerToken = enrollHost(createEnrollment(stranger.id).token, {}).credential;
await expect(settle('commit', removal.id, strangerToken)).rejects.toMatchObject({
status: 403
});
await expect(settle('deposit')).rejects.toMatchObject({ status: 400 });
await expect(settle('commit', randomUUID())).rejects.toMatchObject({ status: 403 });
expect(
await (await settle('commit', removal.id, hostToken, { ...data, server: other })).json()
).toMatchObject({ code: 'wrong_server' });
const first = await (await settle()).json();
expect(first).toMatchObject({ state: 'committed', balance: 2000 });
expect(await (await settle()).json()).toEqual(first);
const cancelled = transfer();
expect(await (await settle('cancel', removal.id, hostToken, cancelled)).json()).toMatchObject({
state: 'cancelled'
});
db().prepare("UPDATE ec_host_operations SET state='completed' WHERE id=?").run(removal.id);
await expect(settle()).rejects.toMatchObject({ status: 403 });
});
it('persists one credit and one ledger row through lost replies, retries and database reopen', async () => {
const data = transfer();
const initial = await post('deposit', data);
expect(await post('deposit', data)).toEqual(initial);
const first = await post('commit', data);
const { closeDb, db } = await import('../../src/lib/server/db/index.js');
closeDb();
expect(await post('commit', data)).toEqual(first);
expect(await post('deposit', data)).toEqual(first);
expect(db().prepare('SELECT amount,balance FROM bank_ledger').all()).toEqual([
{ amount: 2000, balance: 2000 }
]);
expect(await post('deposit', { ...data, amount: 1 })).toMatchObject({
status: 409,
code: 'request_conflict'
});
});
it('shares one GUID balance across games and preserves GUID strings', async () => {
await fund();
expect(await post('balance', { server: other, guid: 'ace' }, otherSecret)).toMatchObject({
balance: 2000
});
const data = transfer('withdraw', 'all', other);
expect(await post('withdraw', data, otherSecret)).toMatchObject({
state: 'prepared',
amount: 2000
});
expect(await post('commit', data, otherSecret)).toMatchObject({
balance: 0,
state: 'committed'
});
const big = { ...transfer(), guid: '18446744073709551615' };
expect(await post('deposit', big)).toMatchObject({ code: 'not_in_match' });
});
it('rejects overdraws, maximum overflow, invalid amounts and point overflow without a ledger change', async () => {
await fund();
for (const [operation, amount, code] of [
['withdraw', 2001, 'insufficient_balance'],
['deposit', 250000, 'bank_limit']
] as const) {
const data = transfer(operation, amount);
const denied = await post(operation, data);
expect(denied).toMatchObject({ ok: false, state: 'rejected', code });
expect(await post(operation, data)).toEqual(denied);
}
expect(await post('withdraw', { ...transfer('withdraw', 'all'), capacity: 100 })).toMatchObject(
{ code: 'points_limit' }
);
for (const amount of [0, -1, 1.5, 1000001, '2000', 'all', null]) {
expect(await post('deposit', { ...transfer(), amount })).toMatchObject({ status: 400 });
}
const { bankProfile } = await import('../../src/lib/server/fleet/bank.js');
expect(bankProfile('ace')).toMatchObject({ balance: 2000, pending: false });
expect(bankProfile('ace').entries).toHaveLength(1);
});
it('requires a present player in the exact current active match', async () => {
expect(await post('deposit', { ...transfer(), guid: 'outsider' })).toMatchObject({
code: 'not_in_match'
});
expect(await post('deposit', { ...transfer(), match: 'old-match' })).toMatchObject({
code: 'not_in_match'
});
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('UPDATE leaderboard_match_players SET present = 0').run();
expect(await post('deposit', transfer())).toMatchObject({ code: 'not_in_match' });
db().prepare('UPDATE leaderboard_match_players SET present = 1').run();
db().prepare("UPDATE leaderboard_matches SET status = 'completed'").run();
expect(await post('deposit', transfer())).toMatchObject({ code: 'not_in_match' });
});
it('cancels uncertain prepares, including cancellation arriving before the prepare', async () => {
await fund();
for (const cancelFirst of [false, true]) {
const data = transfer('withdraw', 500);
if (!cancelFirst) expect(await post('withdraw', data)).toMatchObject({ state: 'prepared' });
const cancelled = await post('cancel', data);
expect(cancelled).toMatchObject({ state: 'cancelled' });
expect(await post('cancel', data)).toEqual(cancelled);
expect(await post('withdraw', data)).toEqual(cancelled);
expect(await post('commit', data)).toEqual(cancelled);
}
const { bankProfile } = await import('../../src/lib/server/fleet/bank.js');
expect(bankProfile('ace')).toMatchObject({ balance: 2000, available: 2000, pending: false });
expect(bankProfile('ace').entries).toHaveLength(1);
});
it('serializes transfers across servers and reserves available funds until settled', async () => {
await fund();
const data = transfer('withdraw', 1500);
await post('withdraw', data);
expect(await post('balance', { server, guid: 'ace' })).toMatchObject({
balance: 2000,
amount: 500,
code: 'pending'
});
expect(await post('withdraw', transfer('withdraw', 1000, other), otherSecret)).toMatchObject({
code: 'pending'
});
await post('commit', data);
expect(await post('balance', { server, guid: 'ace' })).toMatchObject({
balance: 500,
amount: 500
});
});
it('honors prepared deposits after configuration changes and match/server deletion', async () => {
const data = transfer();
await post('deposit', data);
vi.stubEnv('EC_BANK_MAX_BALANCE', '100');
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('DELETE FROM leaderboard_matches').run();
db().prepare('DELETE FROM servers').run();
expect(await post('commit', data)).toMatchObject({ state: 'committed', balance: 2000 });
const { bankProfile } = await import('../../src/lib/server/fleet/bank.js');
expect(bankProfile('ace').entries[0]).toMatchObject({
amount: 2000,
matchAvailable: 0,
server
});
expect(await post('deposit', transfer())).toMatchObject({ code: 'not_in_match' });
expect(await post('cancel', data)).toMatchObject({ state: 'committed' });
});
it('rolls back all writes if the ledger cannot be saved', async () => {
const data = transfer();
await post('deposit', data);
const { db } = await import('../../src/lib/server/db/index.js');
db().exec(
"CREATE TRIGGER reject_bank BEFORE INSERT ON bank_ledger BEGIN SELECT RAISE(ABORT,'test'); END"
);
expect(await post('commit', data)).toMatchObject({ status: 503 });
const { bankProfile } = await import('../../src/lib/server/fleet/bank.js');
expect(bankProfile('ace')).toMatchObject({ balance: 0, pending: true, entries: [] });
db().exec('DROP TRIGGER reject_bank');
expect(await post('commit', data)).toMatchObject({ balance: 2000 });
});
it('returns the original settled result after later transfers change the account', async () => {
const deposited = await fund();
const withdrawal = transfer('withdraw', 500);
await post('withdraw', withdrawal);
expect(await post('commit', withdrawal)).toMatchObject({ balance: 1500 });
expect(await post('commit', deposited)).toMatchObject({ balance: 2000 });
expect(await post('balance', { server, guid: 'ace' })).toMatchObject({ balance: 1500 });
});
it('concurrent preparations cannot reserve the same funds twice', async () => {
await fund();
const first = transfer('withdraw', 2000);
const second = transfer('withdraw', 2000, other);
const results = await Promise.all([
post('withdraw', first),
post('withdraw', second, otherSecret)
]);
expect(results.filter((result) => result.state === 'prepared')).toHaveLength(1);
expect(results.filter((result) => result.code === 'pending')).toHaveLength(1);
const index = results.findIndex((result) => result.state === 'prepared');
expect(
await post('commit', index === 0 ? first : second, index === 0 ? secret : otherSecret)
).toMatchObject({ balance: 0 });
});
@@ -416,3 +416,47 @@ describe('asynchronous native launch preparation', () => {
expect(await operationFor(second.deploymentId)).toBeDefined(); expect(await operationFor(second.deploymentId)).toBeDefined();
}); });
}); });
it('injects unique persistent bank credentials only into private runtime files', async () => {
const job = await queue();
await runPass();
const op = await operationFor(job.deploymentId);
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
const { bankCredential, authenticateBank } = await import('../../src/lib/server/auth/bank.js');
const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
const instance = nativeInstance(job.name)!;
const credential = bankCredential(instance.id);
const payload = getHostOperation(op!.id)!.payload as {
bank: { token: string; configPath: string };
files: { path: string }[];
env: Record<string, string>;
};
expect(payload.bank).toEqual({
token: credential,
configPath: 'configs/electric_cherry_bank.cfg'
});
expect(payload.files.some((file) => file.path.endsWith('electric_cherry_bank.cfg'))).toBe(
false
);
const { db } = await import('../../src/lib/server/db/index.js');
expect(
db()
.prepare('SELECT count(*) AS n FROM file_blobs WHERE instr(content,?) > 0')
.get(Buffer.from(credential))
).toEqual({ n: 0 });
expect(payload.env.PLUTAINER_EXTRA_ARGS).toContain(
'+exec electric_cherry_bank.cfg +set tws_server_id'
);
expect(
authenticateBank(
new Request('https://test', { headers: { Authorization: `Bearer ${credential}` } })
)
).toMatchObject({ ok: true, server: job.name, instanceId: instance.id });
expect(JSON.stringify(recipeForDeployment(job.deploymentId))).not.toContain(credential);
expect(
recipeForDeployment(job.deploymentId)!.recipe.files.some(
(file) => file.path === 'electric_cherry_bank.cfg'
)
).toBe(false);
});
+1
View File
@@ -149,6 +149,7 @@ it.each([
instance: backend.nativeInstance(a.server)!, instance: backend.nativeInstance(a.server)!,
serverKey: 'test-key', serverKey: 'test-key',
rconPassword: password, rconPassword: password,
bankToken: 'ec_bank_' + 'a'.repeat(43),
rebuild: false rebuild: false
}); });
const actualConfig = prepared.recipe.files.find((file) => file.path === target)!; const actualConfig = prepared.recipe.files.find((file) => file.path === target)!;
+61
View File
@@ -91,6 +91,7 @@ it('shows the same ranks in the desktop profile with lifetime map scope explaine
props: { props: {
data: { data: {
profile, profile,
bank: { balance: 0, available: 0, pending: false, entries: [] },
returnTo: '/leaderboard', returnTo: '/leaderboard',
scope: { game: '', map: '', players: '2', metric: 'kills', entry: null } scope: { game: '', map: '', players: '2', metric: 'kills', entry: null }
} }
@@ -135,6 +136,7 @@ it('renders a known player without qualifying history with identity and unavaila
props: { props: {
data: { data: {
profile, profile,
bank: { balance: 0, available: 0, pending: false, entries: [] },
returnTo: '/leaderboard', returnTo: '/leaderboard',
scope: { game: '', map: '', players: '', metric: 'highestRound', entry: null } scope: { game: '', map: '', players: '', metric: 'highestRound', entry: null }
} }
@@ -159,3 +161,62 @@ it('renders a known player without qualifying history with identity and unavaila
expect(body).not.toContain('<table'); expect(body).not.toContain('<table');
expect(body).not.toContain('NaN'); expect(body).not.toContain('NaN');
}); });
it('shows the shared balance and ledger with retained and expired match references', () => {
const profile: PlayerProfile = {
guid: '101',
name: 'Cherry',
firstSeen: evidence.startedAt,
lastSeen: evidence.endedAt!,
stats,
records: map.records,
maps: [],
matches: [],
matchesTotal: 0,
limit: 25,
offset: 0
};
const entry = {
id: 1,
server: evidence.server,
amount: 2000,
balance: 2000,
createdAt: evidence.startedAt,
matchId: 71,
matchAvailable: 1
};
const body = render(PlayerProfilePage, {
props: {
data: {
profile,
bank: {
balance: 1500,
available: 1000,
pending: true,
entries: [
{
...entry,
id: 2,
amount: -500,
balance: 1500,
matchId: 72,
matchAvailable: 0
},
entry
]
},
returnTo: '/leaderboard',
scope: { game: '', map: '', metric: 'highestRound', entry: null }
}
}
}).body;
const text = body.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ');
expect(text).toContain('1,500 points');
expect(text).toContain('shared across Black Ops I and II');
expect(text).toContain('Available: 1,000 points');
expect(text).toContain('+2,000');
expect(text).toContain('-500');
expect(text).toContain('#72 (history expired)');
expect(body).toContain('href="/matches/71"');
expect(body).not.toContain('href="/matches/72"');
});